Repository: sqoop
Updated Branches:
refs/heads/sqoop2 bfcfedfa7 -> 5b897a46f
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
index ae16b85..d527290 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
@@ -161,6 +161,9 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
+ + "cast(null as varchar(100)),"
+ + "false,"
+ + "cast(null as varchar(100)),"
+ "cast(null as varchar(100))"
+ " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
+ " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + " = ?"
@@ -223,7 +226,10 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ","
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
- + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE)
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC)
+ " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
+ " LEFT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID)
@@ -246,7 +252,10 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
- + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE)
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC)
+ " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
+ " LEFT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID)
@@ -271,11 +280,14 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
// DML: Insert new link inputs
private static final String STMT_INSERT_LINK_INPUT =
- "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ " ("
- + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_LINK) + ", "
- + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + ", "
- + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE)
- + ") VALUES (?, ?, ?)";
+ "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ " ("
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_LINK) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC)
+ + ") VALUES (?, ?, ?, ?, ?, ?)";
// DML: Update link
private static final String STMT_UPDATE_LINK =
@@ -385,13 +397,17 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ CommonRepoUtils.escapeColumnName(COLUMN_SQB_UPDATE_DATE)
+ ") VALUES (?, ?, ?, ?, ?, ?, ?, ?)";
+
// DML: Insert new job inputs
private static final String STMT_INSERT_JOB_INPUT =
- "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ " ("
- + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_JOB) + ", "
- + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + ", "
- + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE)
- + ") VALUES (?, ?, ?)";
+ "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ " ("
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_JOB) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC)
+ + ") VALUES (?, ?, ?, ?, ?, ?)";
private static final String STMT_UPDATE_JOB =
"UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME) + " SET "
@@ -741,6 +757,22 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ " IN (SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_ID) + " FROM
" + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIG_NAME)
+ " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_CONFIGURABLE) + "
= ?)";
+ private static final String STMT_SELECT_SQ_MASTER_KEY =
+ "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV)
+ + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME);
+
+ private static final String STMT_INSERT_SQ_MASTER_KEY =
+ "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME)
+ " ("
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + ", "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV)
+ + ") VALUES (?, ?, ?, ?)";
+
+
public String getStmtSelectSqdIdBySqdName() {
return STMT_SELECT_SQD_ID_BY_SQD_NAME;
}
@@ -1042,4 +1074,12 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
public String getStmtInsertIntoInputRelation() {
return STMT_INSERT_INTO_INPUT_RELATION;
}
+
+ public String getStmtSelectSqMasterKey() {
+ return STMT_SELECT_SQ_MASTER_KEY;
+ }
+
+ public String getStmtInsertSqMasterKey() {
+ return STMT_INSERT_SQ_MASTER_KEY;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
index d1940e8..6fd9d6c 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
@@ -175,10 +175,16 @@ public final class CommonRepositorySchemaConstants {
public static final String COLUMN_SQ_LNKI_LINK = "SQ_LNKI_LINK";
+ public static final String COLUMN_SQ_LNKI_ENCRYPTED = "SQ_LNKI_ENCRYPTED";
+
public static final String COLUMN_SQ_LNKI_INPUT = "SQ_LNKI_INPUT";
public static final String COLUMN_SQ_LNKI_VALUE = "SQ_LNKI_VALUE";
+ public static final String COLUMN_SQ_LNKI_IV = "SQ_LNKI_IV";
+
+ public static final String COLUMN_SQ_LNKI_HMAC = "SQ_LNKI_HMAC";
+
// SQ_JOB_INPUT
public static final String TABLE_SQ_JOB_INPUT_NAME = "SQ_JOB_INPUT";
@@ -187,8 +193,14 @@ public final class CommonRepositorySchemaConstants {
public static final String COLUMN_SQBI_INPUT = "SQBI_INPUT";
+ public static final String COLUMN_SQBI_ENCRYPTED = "SQBI_ENCRYPTED";
+
public static final String COLUMN_SQBI_VALUE = "SQBI_VALUE";
+ public static final String COLUMN_SQBI_IV = "SQBI_IV";
+
+ public static final String COLUMN_SQBI_HMAC = "SQBI_HMAC";
+
// SQ_SUBMISSION
public static final String TABLE_SQ_SUBMISSION_NAME = "SQ_SUBMISSION";
@@ -273,6 +285,20 @@ public final class CommonRepositorySchemaConstants {
public static final String COLUMN_SQCO_VALUE = "SQCO_VALUE";
+ // SQ_MASTER_KEY
+
+ public static final String TABLE_SQ_MASTER_KEY_NAME = "SQ_MASTER_KEY";
+
+ public static final String COLUMN_SQMK_ID = "SQMK_ID";
+
+ public static final String COLUMN_SQMK_SECRET = "SQMK_SECRET";
+
+ public static final String COLUMN_SQMK_HMAC = "SQMK_HMAC";
+
+ public static final String COLUMN_SQMK_SALT = "SQMK_SALT";
+
+ public static final String COLUMN_SQMK_IV = "SQMK_IV";
+
// Constraints
public static final String CONSTRAINT_SQCD_SQC_NAME
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index ee5e8d1..75636d4 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -319,6 +319,14 @@ public class DerbyRepositoryHandler extends CommonRepositoryHandler {
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_COLUMN_SQB_NAME, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ALTER_COLUMN_SQB_NAME_NOT_NULL, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIGURABLE_ALTER_COLUMN_SQB_NAME_NOT_NULL, conn);
+
+ runQuery(QUERY_CREATE_TABLE_SQ_MASTER_KEY, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV, conn);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC, conn);
}
// last step upgrade the repository version to the latest value in the code
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
index 1770036..b4f66bb 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
@@ -181,6 +181,10 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* <p>
* <strong>SQ_LINK_INPUT</strong>: N:M relationship link and input
*
+ * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum
length
+ * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where
+ * 1024 is the maximum possible key size in bits for the IV and the digest.
+ *
* <pre>
* +----------------------------+
* | SQ_LINK_INPUT |
@@ -188,6 +192,9 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* | SQ_LNKI_LINK: BIGINT PK | FK SQ_LINK(SQ_LNK_ID)
* | SQ_LNKI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID)
* | SQ_LNKI_VALUE: LONG VARCHAR|
+ * | SQ_LNKI_ENCRYPTED: BOOLEAN |
+ * | SQ_LNKI_IV: VARCHAR(171) |
+ * | SQ_LNKI_HMAC: VARCHAR(171) |
* +----------------------------+
* </pre>
*
@@ -195,6 +202,10 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* <p>
* <strong>SQ_JOB_INPUT</strong>: N:M relationship job and input
*
+ * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum
length
+ * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where
+ * 1024 is the maximum possible key size in bits for the IV and the digest.
+ *
* <pre>
* +----------------------------+
* | SQ_JOB_INPUT |
@@ -202,6 +213,9 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* | SQBI_JOB: BIGINT PK | FK SQ_JOB(SQB_ID)
* | SQBI_INPUT: BIGINT PK | FK SQ_INPUT(SQI_ID)
* | SQBI_VALUE: LONG VARCHAR |
+ * | SQBI_ENCRYPTED: BOOLEAN |
+ * | SQBI_IV: VARCHAR(24) |
+ * | SQBI_HMAC: VARCHAR(171) |
* +----------------------------+
* </pre>
*
@@ -310,6 +324,34 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
* +----------------------------+
* </pre>
* </p>
+ * <p>
+ * <strong>SQ_MASTER_KEY</strong>: Master Key for sensitive value encryption
+ *
+ * SQMK_SECRET length determined by a generous assumption of the maximum length
+ * of the base64 encoded secret. Calculated by ceil((4/3)(1024/8+1024/8)). Where
+ * 1024 is the maximum possible key size in bits for encryption and HMAC.
+ *
+ * SQMK_HMAC length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8))
+ * where 1024 is the maximum possible size of the digest in bits.
+ *
+ * SQMK_SALT length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8))
+ * where 1024 is the maximum possible size of the salt in bits.
+ *
+ * SQMK_IV length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(128))
+ * where 128 is the maximum possible size of the salt in bytes.
+ *
+ * <pre>
+ * +---------------------------+
+ * | SQ_MASTER_KEY |
+ * +---------------------------+
+ * | SQMK_ID: BIGINT PK |
+ * | SQMK_SECRET: VARCHAR(342) |
+ * | SQMK_HMAC: VARCHAR(171) |
+ * | SQMK_SALT: VARCHAR(171) |
+ * | SQMK_IV: VARCHAR(171) |
+ * +---------------------------+
+ * </pre>
+ * </p>
*/
// NOTE: If you have signed yourself to modify the schema for the repository
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
index 5081b82..a427421 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
@@ -662,6 +662,41 @@ public final class DerbySchemaUpgradeQuery {
+ " ALTER COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQC_NAME)
+ " NOT NULL";
+ // DDL: Create table SQ_MASTER_KEY
+ public static final String QUERY_CREATE_TABLE_SQ_MASTER_KEY =
+ "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME)
+ " ("
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + " BIGINT GENERATED ALWAYS AS IDENTITY
(START WITH 1, INCREMENT BY 1), "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + " VARCHAR(342), "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + " VARCHAR(171), "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + " VARCHAR(171), "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) + " VARCHAR(171)"
+ + ")";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + " BOOLEAN
DEFAULT FALSE";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + " VARCHAR(171)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + " VARCHAR(171)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + " BOOLEAN
DEFAULT FALSE";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + " VARCHAR(171)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + " VARCHAR(171)";
+
+
private DerbySchemaUpgradeQuery() {
// Disable explicit object creation
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index e4cca07..9bcfe01 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -247,6 +247,14 @@ abstract public class DerbyTestCase {
runQuery(QUERY_CREATE_TABLE_SQ_CONTEXT_TYPE);
runQuery(QUERY_CREATE_TABLE_SQ_CONTEXT_PROPERTY);
runQuery(QUERY_CREATE_TABLE_SQ_CONTEXT);
+
+ runQuery(QUERY_CREATE_TABLE_SQ_MASTER_KEY);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV);
+ runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC);
}
// deprecated repository version
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
index 2c74c32..87bd6d8 100644
--- a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
+++ b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
@@ -146,6 +146,7 @@ public class MySqlRepositoryHandler extends CommonRepositoryHandler {
runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONTEXT_TYPE, conn);
runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONTEXT_PROPERTY, conn);
runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONTEXT, conn);
+ runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_MASTER_KEY, conn);
// Insert FROM and TO directions.
insertDirections(conn);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
index 47f12fe..29355d5 100644
--- a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
+++ b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
@@ -24,7 +24,7 @@ import org.apache.sqoop.repository.common.CommonRepositorySchemaConstants;
public class MySqlSchemaCreateQuery {
- public static final String QUERY_CREATE_DATABASE_SQOOP = "CREATE DATABASE " + CommonRepoUtils.escapeDatabaseName(SCHEMA_SQOOP);
+ public static final String QUERY_CREATE_DATABASE_SQOOP = "CREATE DATABASE IF NOT EXISTS"
+ CommonRepoUtils.escapeDatabaseName(SCHEMA_SQOOP);
public static final String QUERY_CREATE_TABLE_SQ_SYSTEM =
"CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, CommonRepositorySchemaConstants.TABLE_SQ_SYSTEM_NAME)
+ " ("
@@ -177,6 +177,9 @@ public class MySqlSchemaCreateQuery {
+ CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_LINK)
+ " BIGINT, "
+ CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_INPUT)
+ " BIGINT, "
+ CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_VALUE)
+ " VARCHAR(1000), "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_ENCRYPTED)
+ " BOOLEAN DEFAULT FALSE, "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_IV)
+ " VARCHAR(171), "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_HMAC)
+ " VARCHAR(171),"
+ "PRIMARY KEY (" + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_LINK)
+ ", "
+ CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_INPUT)
+ "), "
+ "CONSTRAINT " + CommonRepoUtils.escapeConstraintName(CommonRepositorySchemaConstants.CONSTRAINT_SQ_LNKI_SQ_LNK_NAME)
@@ -194,6 +197,9 @@ public class MySqlSchemaCreateQuery {
+ CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_JOB)
+ " BIGINT, "
+ CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_INPUT)
+ " BIGINT, "
+ CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_VALUE)
+ " VARCHAR(1000), "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_ENCRYPTED)
+ " BOOLEAN DEFAULT FALSE, "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_IV)
+ " VARCHAR(171), "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_HMAC)
+ " VARCHAR(171),"
+ "PRIMARY KEY (" + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_JOB)
+ ", "
+ CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_INPUT)
+ "), "
+ "CONSTRAINT " + CommonRepoUtils.escapeConstraintName(CommonRepositorySchemaConstants.CONSTRAINT_SQBI_SQB_NAME)
@@ -291,6 +297,16 @@ public class MySqlSchemaCreateQuery {
+ "(" + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQCP_ID)
+ ") "
+ ")";
+ // DDL: Create table SQ_MASTER_KEY
+ public static final String QUERY_CREATE_TABLE_SQ_MASTER_KEY =
+ "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, CommonRepositorySchemaConstants.TABLE_SQ_MASTER_KEY_NAME)
+ " ("
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_ID)
+ " BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_SECRET)
+ " VARCHAR(342), "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_HMAC)
+ " VARCHAR(171), "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_SALT)
+ " VARCHAR(171), "
+ + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_IV)
+ " VARCHAR(171)"
+ + ")";
+
private MySqlSchemaCreateQuery() {
// Disable explicit object creation
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
index 4c295c0..08d3374 100644
--- a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
+++ b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
@@ -36,6 +36,7 @@ public class MysqlRepositoryInsertUpdateDeleteSelectQuery extends
CommonRepositoryInsertUpdateDeleteSelectQuery {
// DML: Get inputs for a given config
+ // MySQL requires that we cast to null to char instead of varchar
private static final String STMT_SELECT_INPUT = "SELECT "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", "
@@ -46,6 +47,9 @@ public class MysqlRepositoryInsertUpdateDeleteSelectQuery extends
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
+ CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
+ + "cast(null as char(100)),"
+ + "false,"
+ + "cast(null as char(100)),"
+ "cast(null as char(100))" + " FROM "
+ CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
+ " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG)
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
index 400d706..b18a7d5 100644
--- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
+++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
@@ -145,6 +145,13 @@ public class PostgresqlRepositoryHandler extends CommonRepositoryHandler
{
runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_COLUMN_SQB_NAME,
conn);
runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ALTER_COLUMN_SQB_NAME_NOT_NULL,
conn);
runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_CONFIGURABLE_ALTER_COLUMN_SQB_NAME_NOT_NULL,
conn);
+ runQuery(PostgresqlSchemaUpgradeQuery.QUERY_CREATE_TABLE_SQ_MASTER_KEY, conn);
+ runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED,
conn);
+ runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV, conn);
+ runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC, conn);
+ runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED,
conn);
+ runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV, conn);
+ runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC,
conn);
}
try (PreparedStatement stmtDel = conn.prepareStatement(PostgresqlSchemaQuery.STMT_DELETE_SYSTEM);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
index 8358df0..e31886e 100644
--- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
+++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
@@ -169,6 +169,11 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants
* </p>
* <p>
* <strong>SQ_LINK_INPUT</strong>: N:M relationship link and input
+ *
+ * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum
length
+ * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where
+ * 1024 is the maximum possible key size in bits for the IV and the digest.
+ *
* <pre>
* +----------------------------+
* | SQ_LINK_INPUT |
@@ -176,11 +181,19 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants
* | SQ_LNK_LINK: BIGSERIAL | FK SQ_LINK(SQ_LNK_ID)
* | SQ_LNK_INPUT: BIGINT | FK SQ_INPUT(SQI_ID)
* | SQ_LNK_VALUE: VARCHAR |
+ * | SQ_LNK_ENCRYPTED: BOOLEAN |
+ * | SQ_LNKI_IV: VARCHAR(171) |
+ * | SQ_LNKI_HMAC: VARCHAR(171) |
* +----------------------------+
* </pre>
* </p>
* <p>
* <strong>SQ_JOB_INPUT</strong>: N:M relationship job and input
+ *
+ * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum
length
+ * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where
+ * 1024 is the maximum possible key size in bits for the IV and the digest.
+ *
* <pre>
* +----------------------------+
* | SQ_JOB_INPUT |
@@ -188,6 +201,9 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants
* | SQBI_JOB: BIGINT | FK SQ_JOB(SQB_ID)
* | SQBI_INPUT: BIGINT | FK SQ_INPUT(SQI_ID)
* | SQBI_VALUE: VARCHAR(1000) |
+ * | SQBI_ENCRYPTED: BOOLEAN |
+ * | SQBI_IV: VARCHAR(171) |
+ * | SQBI_HMAC: VARCHAR(171) |
* +----------------------------+
* </pre>
* </p>
@@ -287,6 +303,34 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants
* +----------------------------+
* </pre>
* </p>
+ * <p>
+ * <strong>SQ_MASTER_KEY</strong>: Master Key for sensitive value encryption
+ *
+ * SQMK_SECRET length determined by a generous assumption of the maximum length
+ * of the base64 encoded secret. Calculated by ceil((4/3)(1024/8+1024/8)). Where
+ * 1024 is the maximum possible key size in bits for encryption and HMAC.
+ *
+ * SQMK_HMAC length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8))
+ * where 1024 is the maximum possible size of the digest in bits.
+ *
+ * SQMK_SALT length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8))
+ * where 1024 is the maximum possible size of the salt in bits.
+ *
+ * SQMK_IV length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(128))
+ * where 128 is the maximum possible size of the salt in bytes.
+ *
+ * <pre>
+ * +---------------------------+
+ * | SQ_MASTER_KEY |
+ * +---------------------------+
+ * | SQMK_ID: BIGINT PK |
+ * | SQMK_SECRET: VARCHAR(342) |
+ * | SQMK_HMAC: VARCHAR(171) |
+ * | SQMK_SALT: VARCHAR(171) |
+ * | SQMK_IV: VARCHAR(171) |
+ * +---------------------------+
+ * </pre>
+ * </p>
*/
public class PostgresqlSchemaCreateQuery {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
index 52954e6..3021039 100644
--- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
+++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
@@ -17,15 +17,30 @@
*/
package org.apache.sqoop.repository.postgresql;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQBI_ENCRYPTED;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQBI_HMAC;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQBI_IV;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQB_ID;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQC_NAME;
+
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_HMAC;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_ID;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_IV;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_SALT;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_SECRET;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_ENCRYPTED;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_HMAC;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_IV;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNK_ID;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNK_NAME;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQB_NAME;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.SCHEMA_SQOOP;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_CONFIGURABLE_NAME;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_JOB_INPUT_NAME;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_JOB_NAME;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_LINK_INPUT_NAME;
import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_LINK_NAME;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_MASTER_KEY_NAME;
import org.apache.sqoop.repository.common.CommonRepoUtils;
@@ -58,4 +73,37 @@ public class PostgresqlSchemaUpgradeQuery {
"ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIGURABLE_NAME)
+ " ALTER COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQC_NAME)
+ " SET NOT NULL";
+
+ public static final String QUERY_CREATE_TABLE_SQ_MASTER_KEY =
+ "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME)
+ " ("
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + " BIGSERIAL PRIMARY KEY NOT NULL,
"
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + " VARCHAR(342), "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + " VARCHAR(171), "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + " VARCHAR(171), "
+ + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) + " VARCHAR(171)"
+ + ")";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + " BOOLEAN
DEFAULT FALSE";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + " VARCHAR(171)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + " VARCHAR(171)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + " BOOLEAN
DEFAULT FALSE";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + " VARCHAR(171)";
+
+ public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC =
+ "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + " VARCHAR(171)";
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
index 00aa214..c2f5b84 100644
--- a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
+++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
@@ -34,7 +34,7 @@ import org.apache.sqoop.server.v1.DriverServlet;
import org.apache.sqoop.server.v1.JobServlet;
import org.apache.sqoop.server.v1.LinkServlet;
import org.apache.sqoop.server.v1.SubmissionsServlet;
-import org.apache.sqoop.utils.ProcessUtils;
+import org.apache.sqoop.utils.PasswordUtils;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
import org.eclipse.jetty.server.SecureRequestCustomizer;
@@ -46,7 +46,6 @@ import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.util.thread.ExecutorThreadPool;
import javax.servlet.DispatcherType;
-import java.io.IOException;
import java.util.EnumSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
@@ -94,40 +93,19 @@ public class SqoopJettyServer {
sslContextFactory.setProtocol(protocol.trim());
}
- String keyStorePassword = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD);
- String keyStorePasswordGenerator = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD_GENERATOR);
- if (StringUtils.isNotBlank(keyStorePassword)) {
- if (StringUtils.isNotBlank(keyStorePasswordGenerator)) {
- LOG.warn(SecurityConstants.KEYSTORE_PASSWORD + " and " + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR
- + "are both set, using " + SecurityConstants.KEYSTORE_PASSWORD);
- }
+ String keyStorePassword = PasswordUtils.readPassword(configurationContext, SecurityConstants.KEYSTORE_PASSWORD,
+ SecurityConstants.KEYSTORE_PASSWORD_GENERATOR);
+ if (StringUtils.isNotEmpty(keyStorePassword)) {
sslContextFactory.setKeyStorePassword(keyStorePassword);
- } else if (StringUtils.isNotBlank(keyStorePasswordGenerator)) {
- try {
- String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyStorePasswordGenerator);
- sslContextFactory.setKeyStorePassword(passwordFromGenerator);
- } catch (IOException exception) {
- throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator:
" + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR, exception);
- }
}
- String keyManagerPassword = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD);
- String keyManagerPasswordGenerator = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR);
- if (StringUtils.isNotBlank(keyManagerPassword)) {
+ String keyManagerPassword = PasswordUtils.readPassword(configurationContext, SecurityConstants.KEYMANAGER_PASSWORD,
+ SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR);
+ if (StringUtils.isNotEmpty(keyManagerPassword)) {
sslContextFactory.setKeyManagerPassword(keyManagerPassword);
- if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) {
- LOG.warn(SecurityConstants.KEYMANAGER_PASSWORD + " and " + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR
- + "are both set, using " + SecurityConstants.KEYMANAGER_PASSWORD);
- }
- } else if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) {
- try {
- String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyManagerPasswordGenerator);
- sslContextFactory.setKeyManagerPassword(passwordFromGenerator);
- } catch (IOException exception) {
- throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator:
" + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR, exception);
- }
}
+
HttpConfiguration https = new HttpConfiguration();
https.addCustomizer(new SecureRequestCustomizer());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java b/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java
index 0b603a1..acd055d 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java
@@ -20,7 +20,7 @@ package org.apache.sqoop.shell;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.OptionBuilder;
import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.utils.ProcessUtils;
+import org.apache.sqoop.utils.PasswordUtils;
import org.apache.sqoop.validation.Status;
import javax.net.ssl.HttpsURLConnection;
@@ -70,7 +70,7 @@ public class SetTruststoreFunction extends SqoopFunction {
truststorePassword = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD).toCharArray();
} else if (line.hasOption(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR)) {
String generator = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR);
- truststorePassword = ProcessUtils.readOutputFromGenerator(generator).toCharArray();
+ truststorePassword = PasswordUtils.readOutputFromGenerator(generator).toCharArray();
}
KeyStore keyStore = KeyStore.getInstance("JKS");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
index c7a4db8..9fa9467 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
@@ -22,6 +22,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.sqoop.core.ConfigurationConstants;
import org.apache.sqoop.common.test.repository.RepositoryProviderFactory;
+import org.apache.sqoop.security.SecurityConstants;
import org.apache.sqoop.test.kdc.KdcRunner;
import java.io.File;
@@ -231,6 +232,16 @@ public abstract class SqoopMiniCluster {
properties.put("org.apache.sqoop.authentication.proxyuser." + "hadoop" + ".groups", "*");
properties.put("org.apache.sqoop.authentication.proxyuser." + "hadoop" + ".hosts", "*");
+ properties.put("org.apache.sqoop.security.repo_encryption.enabled", String.valueOf(true));
+ properties.put("org.apache.sqoop.security.repo_encryption.password_generator", "echo
youwillnevergetthis");
+ properties.put("org.apache.sqoop.security.repo_encryption.hmac_algorithm", "HmacSHA256");
+ properties.put("org.apache.sqoop.security.repo_encryption.cipher_algorithm", "AES");
+ properties.put("org.apache.sqoop.security.repo_encryption.cipher_key_size", "16");
+ properties.put("org.apache.sqoop.security.repo_encryption.initialization_vector_size",
"16");
+ properties.put("org.apache.sqoop.security.repo_encryption.cipher_spec", "AES/CBC/PKCS5Padding");
+ properties.put("org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm", "PBKDF2WithHmacSHA1");
+ properties.put("org.apache.sqoop.security.repo_encryption.pbkdf2_rounds", "4000");
+
return properties;
}
|