knox-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kmin...@apache.org
Subject git commit: KNOX-222: Remove hadoop-examples.jar from source tree
Date Fri, 20 Dec 2013 19:02:19 GMT
Updated Branches:
  refs/heads/master 395de9bc5 -> db3ec3bb8


KNOX-222: Remove hadoop-examples.jar from source tree


Project: http://git-wip-us.apache.org/repos/asf/incubator-knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-knox/commit/db3ec3bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-knox/tree/db3ec3bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-knox/diff/db3ec3bb

Branch: refs/heads/master
Commit: db3ec3bb8060d9cab99a37802de934d357b1b202
Parents: 395de9b
Author: Kevin Minder <kevin.minder@hortonworks.com>
Authored: Fri Dec 20 13:57:30 2013 -0500
Committer: Kevin Minder <kevin.minder@hortonworks.com>
Committed: Fri Dec 20 13:57:30 2013 -0500

----------------------------------------------------------------------
 .../home/samples/ExampleWebHCatJob.groovy       |   2 +-
 .../home/samples/hadoop-examples.jar            | Bin 142457 -> 0 bytes
 gateway-release/pom.xml                         |   5 ++
 gateway-release/src/assembly.xml                |  31 ++++---
 .../hadoop/gateway/GatewayBasicFuncTest.java    |  33 +++++++-
 .../hadoop/gateway/GatewayFuncTestDriver.java   |  13 ++-
 .../GatewayBasicFuncTest/hadoop-examples.jar    | Bin 142457 -> 0 bytes
 hadoop-examples/pom.xml                         |  50 +++++++++++
 .../org/apache/hadoop/examples/WordCount.java   |  84 +++++++++++++++++++
 pom.xml                                         |  15 ++++
 10 files changed, 216 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/gateway-release/home/samples/ExampleWebHCatJob.groovy
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/ExampleWebHCatJob.groovy b/gateway-release/home/samples/ExampleWebHCatJob.groovy
index 50e43fd..0db8497 100644
--- a/gateway-release/home/samples/ExampleWebHCatJob.groovy
+++ b/gateway-release/home/samples/ExampleWebHCatJob.groovy
@@ -45,7 +45,7 @@ session.waitFor( putData, putJar )
 
 jobId = Job.submitJava(session) \
   .jar( jobDir + "/lib/hadoop-examples.jar" ) \
-  .app( "wordcount" ) \
+  .app( "org.apache.hadoop.examples.WordCount" ) \
   .input( jobDir + "/input" ) \
   .output( jobDir + "/output" ) \
   .now().jobId

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/gateway-release/home/samples/hadoop-examples.jar
----------------------------------------------------------------------
diff --git a/gateway-release/home/samples/hadoop-examples.jar b/gateway-release/home/samples/hadoop-examples.jar
deleted file mode 100644
index 351b77a..0000000
Binary files a/gateway-release/home/samples/hadoop-examples.jar and /dev/null differ

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/gateway-release/pom.xml
----------------------------------------------------------------------
diff --git a/gateway-release/pom.xml b/gateway-release/pom.xml
index f165c00..61ad1f4 100644
--- a/gateway-release/pom.xml
+++ b/gateway-release/pom.xml
@@ -180,6 +180,11 @@
 
         <dependency>
             <groupId>${gateway-group}</groupId>
+            <artifactId>hadoop-examples</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>${gateway-group}</groupId>
             <artifactId>gateway-shell</artifactId>
         </dependency>
         <dependency>

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/gateway-release/src/assembly.xml
----------------------------------------------------------------------
diff --git a/gateway-release/src/assembly.xml b/gateway-release/src/assembly.xml
index 77cb291..196ef1b 100644
--- a/gateway-release/src/assembly.xml
+++ b/gateway-release/src/assembly.xml
@@ -35,18 +35,18 @@
                 <exclude>**/.settings/**</exclude>
             </excludes>
         </fileSet>
-		<fileSet>
-		    <directory>..</directory>
-		    <outputDirectory></outputDirectory>
-		    <includes>
-		        <include>CHANGES</include>
-		        <include>DISCLAIMER</include>
-		        <include>ISSUES</include>
-		        <include>LICENSE</include>
-		        <include>NOTICE</include>
-		        <include>README</include>
-		    </includes>
-		</fileSet>
+        <fileSet>
+            <directory>..</directory>
+            <outputDirectory></outputDirectory>
+            <includes>
+                <include>CHANGES</include>
+                <include>DISCLAIMER</include>
+                <include>ISSUES</include>
+                <include>LICENSE</include>
+                <include>NOTICE</include>
+                <include>README</include>
+            </includes>
+        </fileSet>
     </fileSets>
     <dependencySets>
         <dependencySet>
@@ -95,5 +95,12 @@
                 <include>${gateway-group}:gateway-test-ldap-launcher</include>
             </includes>
         </dependencySet>
+        <dependencySet>
+            <outputDirectory>samples</outputDirectory>
+            <outputFileNameMapping>hadoop-examples.jar</outputFileNameMapping>
+            <includes>
+                <include>${gateway-group}:hadoop-examples</include>
+            </includes>
+        </dependencySet>
     </dependencySets>
 </assembly>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
index b8c8ace..c68bbac 100644
--- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java
@@ -21,6 +21,7 @@ import com.jayway.restassured.http.ContentType;
 import com.jayway.restassured.response.Response;
 import com.mycila.xmltool.XMLDoc;
 import com.mycila.xmltool.XMLTag;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.apache.hadoop.test.TestUtils;
 import org.apache.hadoop.test.category.FunctionalTests;
 import org.apache.hadoop.test.category.MediumTests;
@@ -42,6 +43,9 @@ import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.net.InetSocketAddress;
@@ -752,7 +756,7 @@ public class GatewayBasicFuncTest {
     /* Put the mapreduce code into HDFS. (hadoop-examples.jar)
     curl -X PUT --data-binary @hadoop-examples.jar 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/hadoop-examples.jar?user.name=hdfs&op=CREATE'
      */
-    driver.createFile( user, pass, null, root+"/hadoop-examples.jar", "777", "application/octet-stream",
"hadoop-examples.jar", 307, 201, 200 );
+    driver.createFile( user, pass, null, root+"/hadoop-examples.jar", "777", "application/octet-stream",
findHadoopExamplesJar(), 307, 201, 200 );
 
     /* Put the data file into HDFS (changes.txt)
     curl -X PUT --data-binary @changes.txt 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/input/changes.txt?user.name=hdfs&op=CREATE'
@@ -868,7 +872,7 @@ public class GatewayBasicFuncTest {
     /* Put the mapreduce code into HDFS. (hadoop-examples.jar)
     curl -X PUT --data-binary @hadoop-examples.jar 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/hadoop-examples.jar?user.name=hdfs&op=CREATE'
      */
-    driver.createFile( user, pass, group, root+"/lib/hadoop-examples.jar", "777", "application/octet-stream",
"hadoop-examples.jar", 307, 201, 200 );
+    driver.createFile( user, pass, group, root+"/lib/hadoop-examples.jar", "777", "application/octet-stream",
findHadoopExamplesJar(), 307, 201, 200 );
 
     /* Put the data file into HDFS (changes.txt)
     curl -X PUT --data-binary @changes.txt 'http://192.168.1.163:8888/org.apache.org.apache.hadoop.gateway/cluster/webhdfs/v1/user/hdfs/wordcount/input/changes.txt?user.name=hdfs&op=CREATE'
@@ -1978,4 +1982,29 @@ public class GatewayBasicFuncTest {
 //    driver.reset();
     driver.assertComplete();
   }
+
+  private File findFile( File dir, String pattern ) {
+    File file = null;
+    FileFilter filter = new WildcardFileFilter( pattern );
+    File[] files = dir.listFiles(filter);
+    if( files != null && files.length > 0 ) {
+      file = files[0];
+    }
+    return file;
+  }
+
+  private String findHadoopExamplesJar() throws IOException {
+    String pattern = "hadoop-examples-*.jar";
+    File dir = new File( System.getProperty( "user.dir" ), "hadoop-examples/target" );
+    File file = findFile( dir, pattern );
+    if( file == null || !file.exists() ) {
+      file = findFile( new File( System.getProperty( "user.dir" ), "../hadoop-examples/target"
), pattern );
+    }
+    if( file == null ) {
+      throw new FileNotFoundException( pattern );
+    }
+    return file.toURI().toString();
+  }
+
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java
b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java
index 2972258..e043916 100644
--- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java
+++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java
@@ -244,8 +244,17 @@ public class GatewayFuncTestDriver {
     return url;
   }
 
-  public InputStream getResourceStream( String resource ) {
-    InputStream stream = ClassLoader.getSystemResourceAsStream( getResourceName( resource
) );
+  public InputStream getResourceStream( String resource ) throws IOException {
+    InputStream stream = null;
+    if( resource.startsWith( "file:/" ) ) {
+      try {
+        stream = FileUtils.openInputStream( new File( new URI( resource ) ) );
+      } catch( URISyntaxException e ) {
+        throw new IOException( e  );
+      }
+    } else {
+      stream = ClassLoader.getSystemResourceAsStream( getResourceName( resource ) );
+    }
     assertThat( "Failed to find test resource " + resource, stream, Matchers.notNullValue()
);
     return stream;
   }

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/hadoop-examples.jar
----------------------------------------------------------------------
diff --git a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/hadoop-examples.jar
b/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/hadoop-examples.jar
deleted file mode 100644
index 351b77a..0000000
Binary files a/gateway-test/src/test/resources/org/apache/hadoop/gateway/GatewayBasicFuncTest/hadoop-examples.jar
and /dev/null differ

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/hadoop-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-examples/pom.xml b/hadoop-examples/pom.xml
new file mode 100644
index 0000000..79ced74
--- /dev/null
+++ b/hadoop-examples/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>gateway</artifactId>
+        <version>0.4.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>hadoop-examples</artifactId>
+    <name>hadoop-examples</name>
+    <description>Hadoop Examples</description>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-core</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-mapreduce-client-core</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+</project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/hadoop-examples/src/main/java/org/apache/hadoop/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/hadoop-examples/src/main/java/org/apache/hadoop/examples/WordCount.java b/hadoop-examples/src/main/java/org/apache/hadoop/examples/WordCount.java
new file mode 100644
index 0000000..5e4ac4c
--- /dev/null
+++ b/hadoop-examples/src/main/java/org/apache/hadoop/examples/WordCount.java
@@ -0,0 +1,84 @@
+/**
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+package org.apache.hadoop.examples;
+
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+public class WordCount {
+
+  public static class TokenizerMapper
+      extends Mapper<Object, Text, Text, IntWritable>{
+
+    private final static IntWritable one = new IntWritable(1);
+    private Text word = new Text();
+
+    public void map(Object key, Text value, Context context
+    ) throws IOException, InterruptedException {
+      StringTokenizer itr = new StringTokenizer(value.toString());
+      while (itr.hasMoreTokens()) {
+        word.set(itr.nextToken());
+        context.write(word, one);
+      }
+    }
+  }
+
+  public static class IntSumReducer
+      extends Reducer<Text,IntWritable,Text,IntWritable> {
+    private IntWritable result = new IntWritable();
+
+    public void reduce(Text key, Iterable<IntWritable> values,
+                       Context context
+    ) throws IOException, InterruptedException {
+      int sum = 0;
+      for (IntWritable val : values) {
+        sum += val.get();
+      }
+      result.set(sum);
+      context.write(key, result);
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    Configuration conf = new Configuration();
+    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
+    if (otherArgs.length != 2) {
+      System.err.println( "Usage: wordcount <in> <out>" );
+      System.exit(2);
+    }
+    Job job = new Job(conf, "Word Count");
+    job.setJarByClass(WordCount.class);
+    job.setMapperClass(TokenizerMapper.class);
+    job.setCombinerClass(IntSumReducer.class);
+    job.setReducerClass(IntSumReducer.class);
+    job.setOutputKeyClass(Text.class);
+    job.setOutputValueClass(IntWritable.class);
+    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
+    FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
+    System.exit(job.waitForCompletion(true) ? 0 : 1);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-knox/blob/db3ec3bb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 65494e2..d233d2b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -60,6 +60,7 @@
         <module>gateway-service-tgs</module>
         <module>gateway-shell</module>
         <module>gateway-shell-launcher</module>
+        <module>hadoop-examples</module>
         <module>gateway-release</module>
         <module>gateway-test</module>
         <module>hsso-release</module>
@@ -225,6 +226,9 @@
                     <excludedGroups>
                         org.apache.hadoop.test.category.SlowTests,org.apache.hadoop.test.category.ManualTests,org.apache.hadoop.test.category.IntegrationTests
                     </excludedGroups>
+                    <systemPropertyVariables>
+                        <gateway-version>${gateway-version}</gateway-version>
+                    </systemPropertyVariables>
                 </configuration>
             </plugin>
         </plugins>
@@ -451,6 +455,11 @@
             </dependency>
             <dependency>
                 <groupId>${gateway-group}</groupId>
+                <artifactId>hadoop-examples</artifactId>
+                <version>${gateway-version}</version>
+            </dependency>
+            <dependency>
+                <groupId>${gateway-group}</groupId>
                 <artifactId>gateway-release</artifactId>
                 <version>${gateway-version}</version>
             </dependency>
@@ -553,6 +562,12 @@
 
             <dependency>
                 <groupId>org.apache.hadoop</groupId>
+                <artifactId>hadoop-mapreduce-client-core</artifactId>
+                <version>0.23.3</version>
+            </dependency>
+
+            <dependency>
+                <groupId>org.apache.hadoop</groupId>
                 <artifactId>hadoop-core</artifactId>
                 <version>1.0.3</version>
                 <exclusions>


Mime
View raw message