crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-664 Fixes HBase configuration properties being overwritten
Date Tue, 06 Feb 2018 06:00:31 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 8121bdf5a -> 6c0ae4131


CRUNCH-664 Fixes HBase configuration properties being overwritten

Signed-off-by: Josh Wills <jwills@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/6c0ae413
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/6c0ae413
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/6c0ae413

Branch: refs/heads/master
Commit: 6c0ae41318a89a4aed9d343f79849fbadc329e56
Parents: 8121bdf
Author: Nathan Schile <nathan.schile@cerner.com>
Authored: Mon Feb 5 09:08:46 2018 -0600
Committer: Josh Wills <jwills@apache.org>
Committed: Mon Feb 5 21:40:50 2018 -0800

----------------------------------------------------------------------
 .../org/apache/crunch/io/hbase/HBaseTarget.java |  2 +-
 .../org/apache/crunch/io/hbase/HFileTarget.java |  2 +-
 .../apache/crunch/io/hbase/HBaseTargetTest.java | 41 ++++++++++++++++++++
 .../apache/crunch/io/hbase/HFileTargetTest.java | 41 ++++++++++++++++++++
 .../src/test/resources/test-hbase-conf.xml      |  8 ++++
 5 files changed, 92 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java
index f287d5e..f4f134d 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HBaseTarget.java
@@ -100,7 +100,7 @@ public class HBaseTarget implements MapReduceTarget {
   @Override
   public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String
name) {
     final Configuration conf = job.getConfiguration();
-    HBaseConfiguration.addHbaseResources(conf);
+    HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
     conf.setStrings("io.serializations", conf.get("io.serializations"),
         MutationSerialization.class.getName());
     Class<?> typeClass = ptype.getTypeClass(); // Either Put or Delete

http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java
index 41d56ff..8593a76 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileTarget.java
@@ -56,7 +56,7 @@ public class HFileTarget extends FileTargetImpl {
   @Override
   public void configureForMapReduce(Job job, PType<?> ptype, Path outputPath, String
name) {
     Configuration conf = job.getConfiguration();
-    HBaseConfiguration.addHbaseResources(conf);
+    HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
     conf.setStrings("io.serializations", conf.get("io.serializations"),
         KeyValueSerialization.class.getName());
     super.configureForMapReduce(job, ptype, outputPath, name);

http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java
new file mode 100644
index 0000000..8faa27d
--- /dev/null
+++ b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HBaseTargetTest.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.io.hbase;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class HBaseTargetTest {
+
+  @Test
+  public void testConfigureForMapReduce() throws IOException {
+    Job job = Job.getInstance();
+    // Add the test config file. We can't just call job.getConfiguration().set() because
+    // setting a configuration with .set will always to precedence.
+    job.getConfiguration().addResource("test-hbase-conf.xml");
+
+    HBaseTarget target = new HBaseTarget("testTable");
+    target.configureForMapReduce(job, HBaseTypes.keyValues(), new Path("/"), "name");
+
+    assertEquals("12345", job.getConfiguration().get("hbase.client.scanner.timeout.period"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java
new file mode 100644
index 0000000..fbe1a72
--- /dev/null
+++ b/crunch-hbase/src/test/java/org/apache/crunch/io/hbase/HFileTargetTest.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.io.hbase;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+public class HFileTargetTest {
+
+  @Test
+  public void testConfigureForMapReduce() throws IOException {
+    Job job = Job.getInstance();
+    // Add the test config file. We can't just call job.getConfiguration().set() because
+    // setting a configuration with .set will always to precedence.
+    job.getConfiguration().addResource("test-hbase-conf.xml");
+
+    HFileTarget target = new HFileTarget("/");
+    target.configureForMapReduce(job, HBaseTypes.keyValues(), new Path("/"), "name");
+
+    assertEquals("12345", job.getConfiguration().get("hbase.client.scanner.timeout.period"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/6c0ae413/crunch-hbase/src/test/resources/test-hbase-conf.xml
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/test/resources/test-hbase-conf.xml b/crunch-hbase/src/test/resources/test-hbase-conf.xml
new file mode 100644
index 0000000..a6e6dc0
--- /dev/null
+++ b/crunch-hbase/src/test/resources/test-hbase-conf.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>hbase.client.scanner.timeout.period</name>
+    <value>12345</value>
+  </property>
+</configuration>


Mime
View raw message