crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [crunch] 02/03: Ever more fixes
Date Thu, 14 Nov 2019 21:54:24 GMT
This is an automated email from the ASF dual-hosted git repository.

jwills pushed a commit to branch jwills_great_version_upgrade
in repository https://gitbox.apache.org/repos/asf/crunch.git

commit 7c677940061534f17ecaf3ef5efcd56ee07984e9
Author: Josh Wills <jwills@apache.org>
AuthorDate: Thu Nov 14 13:50:15 2019 -0800

    Ever more fixes
---
 .../src/main/resources/archetype-resources/pom.xml |  4 ++--
 .../io/parquet/AvroParquetFileSourceTargetIT.java  |  2 +-
 .../types/avro/AvroChildClassloaderTestRunner.java | 22 ++++++++++++++++---
 .../org/apache/crunch/types/avro/AvroModeTest.java | 25 ----------------------
 pom.xml                                            | 15 +++++++------
 5 files changed, 30 insertions(+), 38 deletions(-)

diff --git a/crunch-archetype/src/main/resources/archetype-resources/pom.xml b/crunch-archetype/src/main/resources/archetype-resources/pom.xml
index 6575417..34da675 100644
--- a/crunch-archetype/src/main/resources/archetype-resources/pom.xml
+++ b/crunch-archetype/src/main/resources/archetype-resources/pom.xml
@@ -89,8 +89,8 @@
         <artifactId>maven-compiler-plugin</artifactId>
         <version>2.5.1</version>
         <configuration>
-          <source>1.6</source>
-          <target>1.6</target>
+          <source>1.8</source>
+          <target>1.8</target>
         </configuration>
       </plugin>
       <plugin>
diff --git a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
index 1b2b151..3c80a3c 100644
--- a/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/io/parquet/AvroParquetFileSourceTargetIT.java
@@ -153,7 +153,7 @@ public class AvroParquetFileSourceTargetIT implements Serializable {
 
     Person person = Iterables.getOnlyElement(ageOnly.materialize());
     assertNull(person.getName());
-    assertEquals(person.getAge(), 42);
+    assertEquals(person.getAge().intValue(), 42);
     assertNull(person.getSiblingnames());
   }
 
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java
b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java
index a1f93b7..d0016fc 100644
--- a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java
+++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroChildClassloaderTestRunner.java
@@ -17,6 +17,8 @@
  */
 package org.apache.crunch.types.avro;
 
+import java.io.File;
+import java.lang.management.ManagementFactory;
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.util.ArrayList;
@@ -45,11 +47,25 @@ public class AvroChildClassloaderTestRunner extends BlockJUnit4ClassRunner
{
     private static ClassLoader parentClassLoader;
     private static URL[] crunchURLs;
     static {
-      URLClassLoader systemClassLoader = (URLClassLoader) getSystemClassLoader();
+      ClassLoader classLoader = getSystemClassLoader();
+      URL[] urls = null;
+      if (classLoader instanceof URLClassLoader) {
+	 urls = ((URLClassLoader) classLoader).getURLs();
+      } else {
+        String[] pieces = ManagementFactory.getRuntimeMXBean().getClassPath().split(File.pathSeparator);
+        urls = new URL[pieces.length];
+        for (int i = 0; i < pieces.length; i++) {
+          try {
+            urls[i] = new File(pieces[i]).toURI().toURL();
+          } catch (Exception e) {
+            throw new RuntimeException(e);
+          }
+        }
+      }
 
       Collection<URL> crunchURLs = new ArrayList<URL>();
       Collection<URL> otherURLs = new ArrayList<URL>();
-      for (URL url : systemClassLoader.getURLs()) {
+      for (URL url : urls) {
         if (url.getPath().matches("^.*/crunch-?.*/.*$")) {
           crunchURLs.add(url);
         } else {
@@ -58,7 +74,7 @@ public class AvroChildClassloaderTestRunner extends BlockJUnit4ClassRunner
{
       }
 
       TestClassLoader.crunchURLs = crunchURLs.toArray(new URL[crunchURLs.size()]);
-      parentClassLoader = new URLClassLoader(otherURLs.toArray(new URL[otherURLs.size()]),
null);
+      parentClassLoader = new URLClassLoader(otherURLs.toArray(new URL[otherURLs.size()]),
classLoader);
     }
 
     public TestClassLoader() {
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroModeTest.java b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroModeTest.java
index fe36520..8df49d3 100644
--- a/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroModeTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/types/avro/AvroModeTest.java
@@ -23,7 +23,6 @@ import static org.hamcrest.CoreMatchers.not;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThat;
-import static org.mockito.Mockito.mock;
 
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericData;
@@ -152,30 +151,6 @@ public class AvroModeTest {
     assertThat(returnedMode.getFactory(), is(instanceOf(FakeReaderWriterFactory.class)));
   }
 
-  @Test
-  public void testRegisterClassLoader() {
-    // First make sure things are in the default situation
-    AvroMode.setSpecificClassLoader(null);
-
-    ClassLoader classLoaderA = mock(ClassLoader.class);
-    ClassLoader classLoaderB = mock(ClassLoader.class);
-
-    // Basic sanity check to ensure that the class loader was really nulled out
-    assertNull(AvroMode.getSpecificClassLoader());
-
-    // Do an internal registration of a class loader. Because there is currently no internal
class loader set,
-    // this should set the internal specific class loader
-    AvroMode.registerSpecificClassLoaderInternal(classLoaderA);
-
-    assertEquals(classLoaderA, AvroMode.getSpecificClassLoader());
-
-    // Now we do an internal register of another class loader. Because there already is an
internal specific class
-    // loader set, this should have no impact (as opposed to calling setSpecificClassLoader)
-    AvroMode.registerSpecificClassLoaderInternal(classLoaderB);
-
-    assertEquals(classLoaderA, AvroMode.getSpecificClassLoader());
-  }
-
   private static class FakeReaderWriterFactory implements ReaderWriterFactory{
 
     @Override
diff --git a/pom.xml b/pom.xml
index 05a21ad..9b8b526 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,9 +85,9 @@ under the License.
     <commons-lang.version>2.6</commons-lang.version>
     <commons-codec.version>1.4</commons-codec.version>
     <commons-cli.version>1.2</commons-cli.version>
-    <avro.version>1.9.1</avro.version>
+    <avro.version>1.8.2</avro.version>
     <hive.version>2.1.0</hive.version>
-    <parquet.version>1.8.1</parquet.version>
+    <parquet.version>1.10.1</parquet.version>
     <javassist.version>3.16.1-GA</javassist.version>
     <jackson.databind.version>2.6.1</jackson.databind.version>
     <protobuf-java.version>2.5.0</protobuf-java.version>
@@ -106,7 +106,7 @@ under the License.
 
     <kafka.version>1.1.0</kafka.version>
     <scala.base.version>2.12</scala.base.version>
-    <scala.version>2.12.6</scala.version>
+    <scala.version>2.12.10</scala.version>
     <scalatest.version>3.0.1</scalatest.version>
     <spark.version>2.4.0</spark.version>
     <jline.version>2.12.1</jline.version>
@@ -278,7 +278,7 @@ under the License.
         <exclusions>
           <exclusion>
             <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-core</artifactId>
+            <artifactId>hadoop-client</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
@@ -641,7 +641,7 @@ under the License.
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
-          <version>3.3</version>
+          <version>3.8.1</version>
           <configuration>
             <source>${java.source.version}</source>
             <target>${java.target.version}</target>
@@ -689,7 +689,7 @@ under the License.
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-assembly-plugin</artifactId>
-          <version>2.5.4</version>
+          <version>3.1.1</version>
         </plugin>
         <plugin>
           <groupId>net.alchim31.maven</groupId>
@@ -701,6 +701,7 @@ under the License.
           <artifactId>maven-surefire-plugin</artifactId>
           <version>2.22.2</version>
           <configuration>
+            <useSystemClassLoader>false</useSystemClassLoader>
             <argLine>-Xmx2G -Dfile.encoding=UTF-8</argLine>
           </configuration>
         </plugin>
@@ -712,7 +713,7 @@ under the License.
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-dependency-plugin</artifactId>
-          <version>2.22.2</version>
+          <version>2.10</version>
         </plugin>
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>


Mime
View raw message