crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [crunch] 01/03: WIP for modernizing Crunch deps
Date Thu, 14 Nov 2019 21:54:23 GMT
This is an automated email from the ASF dual-hosted git repository.

jwills pushed a commit to branch jwills_great_version_upgrade
in repository https://gitbox.apache.org/repos/asf/crunch.git

commit d94800c556368829870a9bac64618dfe65d5699d
Author: Josh Wills <jwills@apache.org>
AuthorDate: Wed Nov 13 15:14:51 2019 -0800

    WIP for modernizing Crunch deps
---
 crunch-core/pom.xml                                |  9 +++--
 .../io/parquet/AvroParquetFileSourceTargetIT.java  |  2 +-
 .../crunch/io/parquet/AvroParquetFileSource.java   |  2 +-
 .../crunch/util/CrunchRenameCopyListing.java       |  4 +--
 .../parquet/AvroParquetFileReaderFactoryTest.java  |  2 +-
 crunch-kafka/pom.xml                               |  2 +-
 .../java/org/apache/crunch/kafka/KafkaUtils.java   |  2 +-
 .../scrunch/spark/ByteBufferInputStream.scala      |  2 +-
 crunch-test/pom.xml                                |  2 +-
 pom.xml                                            | 41 ++++++++++++----------
 10 files changed, 38 insertions(+), 30 deletions(-)

diff --git a/crunch-core/pom.xml b/crunch-core/pom.xml
index 1f487a2..26a0707 100644
--- a/crunch-core/pom.xml
+++ b/crunch-core/pom.xml
@@ -42,7 +42,6 @@ under the License.
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>
-      <classifier>${avro.classifier}</classifier>
     </dependency>
 
     <dependency>
@@ -69,6 +68,12 @@ under the License.
 
     <dependency>
       <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-distcp</artifactId>
       <scope>provided</scope>
     </dependency>
@@ -142,7 +147,7 @@ under the License.
 
     <dependency>
       <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
+      <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
 
diff --git a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
index a14a5e3..1b2b151 100644
--- a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
@@ -153,7 +153,7 @@ public class AvroParquetFileSourceTargetIT implements Serializable {
 
     Person person = Iterables.getOnlyElement(ageOnly.materialize());
     assertNull(person.getName());
-    assertEquals(person.getAge(), new Integer(42));
+    assertEquals(person.getAge(), 42);
     assertNull(person.getSiblingnames());
   }
 
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
index 09703bd..9660b16 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/parquet/AvroParquetFileSource.java
@@ -186,7 +186,7 @@ public class AvroParquetFileSource<T extends IndexedRecord> extends
FileSourceIm
       if (field == null) {
         throw new IllegalArgumentException("No field " + fieldName + " in schema: " + baseSchema.getName());
       }
-      fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue(),
field.order()));
+      fields.add(new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal(),
field.order()));
       return this;
     }
 
diff --git a/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java
b/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java
index b930beb..7431d5d 100644
--- a/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java
+++ b/crunch-core/src/main/java/org/apache/crunch/util/CrunchRenameCopyListing.java
@@ -219,7 +219,7 @@ public class CrunchRenameCopyListing extends SimpleCopyListing {
           DistCpUtils.getRelativePath(sourcePathRoot, fileStatus.getPath()), fileStatus.getPath());
     }
 
-    if (!shouldCopy(fileStatus.getPath(), options)) {
+    if (!shouldCopy(fileStatus.getPath())) {
       return;
     }
 
@@ -269,4 +269,4 @@ public class CrunchRenameCopyListing extends SimpleCopyListing {
   protected long getNumberOfPaths() {
     return totalPaths;
   }
-}
\ No newline at end of file
+}
diff --git a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
index d2ad5bc..95d4561 100644
--- a/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/io/parquet/AvroParquetFileReaderFactoryTest.java
@@ -98,7 +98,7 @@ public class AvroParquetFileReaderFactoryTest {
   }
 
   public static Schema.Field cloneField(Schema.Field field) {
-    return new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultValue());
+    return new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal());
   }
 
 }
diff --git a/crunch-kafka/pom.xml b/crunch-kafka/pom.xml
index 14817b6..1036808 100644
--- a/crunch-kafka/pom.xml
+++ b/crunch-kafka/pom.xml
@@ -40,7 +40,7 @@ under the License.
     </dependency>
     <dependency>
       <groupId>org.apache.kafka</groupId>
-      <artifactId>kafka_2.11</artifactId>
+      <artifactId>kafka_${scala.base.version}</artifactId>
     </dependency>
     <dependency>
       <groupId>org.scala-lang</groupId>
diff --git a/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java b/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java
index 2681df0..0e9d750 100644
--- a/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java
+++ b/crunch-kafka/src/main/java/org/apache/crunch/kafka/KafkaUtils.java
@@ -34,7 +34,7 @@ import org.apache.crunch.CrunchRuntimeException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.kafka.common.TopicPartition;
 import org.apache.kafka.common.network.ListenerName;
-import org.apache.kafka.common.protocol.SecurityProtocol;
+import org.apache.kafka.common.security.auth.SecurityProtocol; 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import scala.Option;
diff --git a/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala
b/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala
index cc11b46..b43fa55 100644
--- a/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala
+++ b/crunch-spark/src/main/scala/org/apache/crunch/scrunch/spark/ByteBufferInputStream.scala
@@ -55,7 +55,7 @@ class ByteBufferInputStream(private var buffer: ByteBuffer)
   override def skip(bytes: Long): Long = {
     if (buffer != null) {
       val amountToSkip = math.min(bytes, buffer.remaining).toInt
-      buffer.position(buffer.position + amountToSkip)
+      buffer.position(buffer.position() + amountToSkip)
       if (buffer.remaining() == 0) {
         cleanUp()
       }
diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml
index b2c18dc..fb3110d 100644
--- a/crunch-test/pom.xml
+++ b/crunch-test/pom.xml
@@ -68,7 +68,7 @@ under the License.
     
     <dependency>
        <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
+      <artifactId>mockito-core</artifactId>
     </dependency>
 
     <dependency>
diff --git a/pom.xml b/pom.xml
index 46d8838..05a21ad 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,8 +72,8 @@ under the License.
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-    <java.source.version>1.7</java.source.version>
-    <java.target.version>1.7</java.target.version>
+    <java.source.version>1.8</java.source.version>
+    <java.target.version>1.8</java.target.version>
     <!--
         NOTE: Please don't change any dependency versions here without
               checking first that they match those on Hadoop's runtime
@@ -85,7 +85,7 @@ under the License.
     <commons-lang.version>2.6</commons-lang.version>
     <commons-codec.version>1.4</commons-codec.version>
     <commons-cli.version>1.2</commons-cli.version>
-    <avro.version>1.7.7</avro.version>
+    <avro.version>1.9.1</avro.version>
     <hive.version>2.1.0</hive.version>
     <parquet.version>1.8.1</parquet.version>
     <javassist.version>3.16.1-GA</javassist.version>
@@ -94,22 +94,21 @@ under the License.
     <libthrift.version>0.8.0</libthrift.version>
     <slf4j.version>1.6.1</slf4j.version>
     <log4j.version>1.2.15</log4j.version>
-    <algebird.version>0.9.0</algebird.version>
+    <algebird.version>0.13.2</algebird.version>
     <junit.version>4.10</junit.version>
     <hamcrest.version>1.1</hamcrest.version>
-    <mockito.version>1.9.0</mockito.version>
+    <mockito.version>2.23.4</mockito.version>
     <pkg>org.apache.crunch</pkg>
 
-    <hadoop.version>2.7.1</hadoop.version>
+    <hadoop.version>2.8.2</hadoop.version>
     <hbase.version>2.0.1</hbase.version>
-    <avro.classifier>hadoop2</avro.classifier>
     <hive.version>2.1.0</hive.version>
 
-    <kafka.version>0.10.2.1</kafka.version>
-    <scala.base.version>2.11</scala.base.version>
-    <scala.version>2.11.8</scala.version>
-    <scalatest.version>2.2.4</scalatest.version>
-    <spark.version>2.0.0</spark.version>
+    <kafka.version>1.1.0</kafka.version>
+    <scala.base.version>2.12</scala.base.version>
+    <scala.version>2.12.6</scala.version>
+    <scalatest.version>3.0.1</scalatest.version>
+    <spark.version>2.4.0</spark.version>
     <jline.version>2.12.1</jline.version>
     <jsr305.version>1.3.9</jsr305.version>
   </properties>
@@ -202,6 +201,12 @@ under the License.
 
       <dependency>
         <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-hdfs</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
+
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-minicluster</artifactId>
         <version>${hadoop.version}</version>
       </dependency>
@@ -258,7 +263,6 @@ under the License.
         <groupId>org.apache.avro</groupId>
         <artifactId>avro-mapred</artifactId>
         <version>${avro.version}</version>
-        <classifier>${avro.classifier}</classifier>
         <exclusions>
           <exclusion>
             <groupId>org.apache.avro</groupId>
@@ -391,7 +395,7 @@ under the License.
     
       <dependency>
         <groupId>org.mockito</groupId>
-        <artifactId>mockito-all</artifactId>
+        <artifactId>mockito-core</artifactId>
         <version>${mockito.version}</version>
       </dependency>
            
@@ -695,9 +699,9 @@ under the License.
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-surefire-plugin</artifactId>
-          <version>2.18.1</version>
+          <version>2.22.2</version>
           <configuration>
-            <argLine>-Xmx2G -XX:PermSize=512m -XX:MaxPermSize=1G -Dfile.encoding=UTF-8</argLine>
+            <argLine>-Xmx2G -Dfile.encoding=UTF-8</argLine>
           </configuration>
         </plugin>
         <plugin>
@@ -708,7 +712,7 @@ under the License.
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-dependency-plugin</artifactId>
-          <version>2.10</version>
+          <version>2.22.2</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
@@ -837,9 +841,8 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-failsafe-plugin</artifactId>
-        <version>2.18.1</version>
+        <version>2.22.2</version>
         <configuration>
-          <argLine>-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m -Dfile.encoding=UTF-8</argLine>
           <testSourceDirectory>${basedir}/src/it/java</testSourceDirectory>
         </configuration>
         <executions>


Mime
View raw message