yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject yetus git commit: YETUS-281. hadoop: use built-in dependency order
Date Tue, 09 Feb 2016 00:40:09 GMT
Repository: yetus
Updated Branches:
  refs/heads/master 57d69414a -> 21f567784


YETUS-281. hadoop: use built-in dependency order

Signed-off-by: Chris Nauroth <cnauroth@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/21f56778
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/21f56778
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/21f56778

Branch: refs/heads/master
Commit: 21f567784073c45ec650ca79f281ba7afedde460
Parents: 57d6941
Author: Allen Wittenauer <aw@apache.org>
Authored: Sat Feb 6 05:52:07 2016 -0800
Committer: Allen Wittenauer <aw@apache.org>
Committed: Mon Feb 8 16:39:39 2016 -0800

----------------------------------------------------------------------
 precommit/personality/hadoop.sh | 219 +++++++++--------------------------
 1 file changed, 53 insertions(+), 166 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/yetus/blob/21f56778/precommit/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/hadoop.sh b/precommit/personality/hadoop.sh
index 60944e8..9d6f6cd 100755
--- a/precommit/personality/hadoop.sh
+++ b/precommit/personality/hadoop.sh
@@ -30,169 +30,32 @@ function personality_globals
   GITHUB_REPO="apache/hadoop"
   #shellcheck disable=SC2034
   PYLINT_OPTIONS="--indent-string='  '"
-
-  HADOOP_MODULES=""
 }
 
-function hadoop_module_manipulation
+function hadoop_unittest_prereqs
 {
-  local startingmodules=${1:-normal}
-  local full_ordered_hadoop_modules;
-  local module
-  local ordered_modules
-  local passed_modules
-  local flags
-
-  yetus_debug "hmm in: ${startingmodules}"
-
-  if [[ ${startingmodules} = normal ]]; then
-    startingmodules=${CHANGED_MODULES}
-  elif  [[ ${startingmodules} = union ]]; then
-    startingmodules=${CHANGED_UNION_MODULES}
-  fi
-
-  yetus_debug "hmm expanded to: ${startingmodules}"
-
-  # If "." is present along with other changed modules,
-  # then just choose "."
-  for module in ${startingmodules}; do
-    if [[ "${module}" = "." ]]; then
-      yetus_debug "hmm shortcut since ."
-      HADOOP_MODULES=.
-      return
-    fi
-  done
-
-  passed_modules=${startingmodules}
-
-  yetus_debug "hmm pre-ordering: ${startingmodules}"
+  declare mods=("$@")
+  declare need_common=0
+  declare building_common=0
+  declare module
+  declare flags
+  declare fn
 
-  # Build a full ordered list of modules
-  # based in maven dependency and re-arrange changed modules
-  # in dependency order.
-  # NOTE: module names with spaces not expected in hadoop
-  full_ordered_hadoop_modules=(
-    hadoop-build-tools
-    hadoop-project
-    hadoop-common-project/hadoop-annotations
-    hadoop-project-dist
-    hadoop-assemblies
-    hadoop-maven-plugins
-    hadoop-common-project/hadoop-minikdc
-    hadoop-common-project/hadoop-auth
-    hadoop-common-project/hadoop-auth-examples
-    hadoop-common-project/hadoop-common
-    hadoop-common-project/hadoop-nfs
-    hadoop-common-project/hadoop-kms
-    hadoop-common-project
-    hadoop-hdfs-project/hadoop-hdfs-client
-    hadoop-hdfs-project/hadoop-hdfs
-    hadoop-hdfs-project/hadoop-hdfs-native-client
-    hadoop-hdfs-project/hadoop-hdfs-httfs
-    hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal
-    hadoop-hdfs-project/hadoop-hdfs-nfs
-    hadoop-hdfs-project
-    hadoop-yarn-project/hadoop-yarn
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site
-    hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry
-    hadoop-yarn-project
-    hadoop-mapreduce-project/hadoop-mapreduce-client
-    hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core
-    hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common
-    hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle
-    hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app
-    hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs
-    hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient
-    hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins
-    hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask
-    hadoop-mapreduce-project/hadoop-mapreduce-examples
-    hadoop-mapreduce-project
-    hadoop-tools/hadoop-streaming
-    hadoop-tools/hadoop-distcp
-    hadoop-tools/hadoop-archives
-    hadoop-tools/hadoop-archive-logs
-    hadoop-tools/hadoop-rumen
-    hadoop-tools/hadoop-gridmix
-    hadoop-tools/hadoop-datajoin
-    hadoop-tools/hadoop-ant
-    hadoop-tools/hadoop-extras
-    hadoop-tools/hadoop-pipes
-    hadoop-tools/hadoop-openstack
-    hadoop-tools/hadoop-aws
-    hadoop-tools/hadoop-azure
-    hadoop-client
-    hadoop-minicluster
-    hadoop-tools/hadoop-sls
-    hadoop-tools/hadoop-tools-dist
-    hadoop-tools/hadoop-kafka
-    hadoop-tools
-    hadoop-dist)
+  # prior to running unit tests, hdfs needs libhadoop.so built
+  # if we're building root, then this extra work is moot
 
-  # For each expected ordered module,
-  # if it's in changed modules, add to HADOOP_MODULES
-  for module in "${full_ordered_hadoop_modules[@]}"; do
-    # shellcheck disable=SC2086
-    if hadoop_check_module_present "${module}" ${passed_modules}; then
-      yetus_debug "Personality ordering ${module}"
-      ordered_modules="${ordered_modules} ${module}"
-    fi
-  done
-
-  # For modules which are not in ordered list,
-  # add them at last
-  for module in $( echo "${passed_modules}" | tr ' ' '\n'); do
-    # shellcheck disable=SC2086
-    if ! hadoop_check_module_present "${module}" ${ordered_modules}; then
-      yetus_debug "Personality ordering ${module}"
-      ordered_modules="${ordered_modules} ${module}"
-    fi
-  done
-
-  HADOOP_MODULES="${ordered_modules}"
-
-  yetus_debug "hmm out: ${HADOOP_MODULES}"
-}
+  # if module handling is ever put into arrays (YETUS-300)
+  # undo this disable and quote the array
 
-function hadoop_check_module_present
-{
-  local module_to_check=${1}
-  shift
-  for module in "${@}"; do
-    if [[ ${module_to_check} = "${module}" ]]; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-function hadoop_unittest_prereqs
-{
-  local need_common=0
-  local building_common=0
-  local module
-  local flags
-  local fn
-
-  for module in ${HADOOP_MODULES}; do
+  #shellcheck disable=SC2068
+  for module in ${mods[@]}; do
     if [[ ${module} = hadoop-hdfs-project* ]]; then
       need_common=1
     elif [[ ${module} = hadoop-common-project/hadoop-common
       || ${module} = hadoop-common-project ]]; then
       building_common=1
+    elif [[ ${module} = . ]]; then
+      return
     fi
   done
 
@@ -261,14 +124,14 @@ function hadoop_native_flags
 
 function personality_modules
 {
-  local repostatus=$1
-  local testtype=$2
-  local extra=""
-  local ordering="normal"
-  local needflags=false
-  local flags
-  local fn
-  local i
+  declare repostatus=$1
+  declare testtype=$2
+  declare extra=""
+  declare ordering="normal"
+  declare needflags=false
+  declare flags
+  declare fn
+  declare i
 
   yetus_debug "Personality: ${repostatus} ${testtype}"
 
@@ -295,15 +158,19 @@ function personality_modules
         yetus_debug "hadoop personality: javac + hadoop-common = ordering set to . "
         ordering="."
       fi
-      ;;
+    ;;
     distclean)
       ordering="."
       extra="-DskipTests"
     ;;
     javadoc)
+      if [[ "${CHANGED_MODULES}" =~ \. ]]; then
+        ordering=.
+      fi
+
       if [[ ${repostatus} = patch ]]; then
         echo "javadoc pre-reqs:"
-        for i in  hadoop-project \
+        for i in hadoop-project \
           hadoop-common-project/hadoop-annotations; do
             fn=$(module_file_fragment "${i}")
             pushd "${BASEDIR}/${i}" >/dev/null
@@ -315,13 +182,27 @@ function personality_modules
       fi
       extra="-Pdocs -DskipTests"
     ;;
+    mvneclipse)
+      if [[ "${CHANGED_MODULES}" =~ \. ]]; then
+        ordering=.
+      fi
+    ;;
     mvninstall)
       extra="-DskipTests"
       if [[ ${repostatus} = branch ]]; then
         ordering=.
       fi
-      ;;
+    ;;
+    mvnsite)
+      if [[ "${CHANGED_MODULES}" =~ \. ]]; then
+        ordering=.
+      fi
+    ;;
     unit)
+      if [[ "${CHANGED_MODULES}" =~ \. ]]; then
+        ordering=.
+      fi
+
       if [[ ${TEST_PARALLEL} = "true" ]] ; then
         extra="-Pparallel-tests"
         if [[ -n ${TEST_THREADS:-} ]]; then
@@ -329,7 +210,7 @@ function personality_modules
         fi
       fi
       needflags=true
-      hadoop_unittest_prereqs
+      hadoop_unittest_prereqs "${ordering}"
 
       verify_needed_test javac
       if [[ $? == 0 ]]; then
@@ -363,7 +244,13 @@ function personality_modules
 
   extra="-Ptest-patch ${extra}"
 
-  hadoop_module_manipulation ${ordering}
+  if [[ ${ordering} = normal ]]; then
+    HADOOP_MODULES=${CHANGED_MODULES}
+  elif  [[ ${ordering} = union ]]; then
+    HADOOP_MODULES=${CHANGED_UNION_MODULES}
+  else
+    HADOOP_MODULES="${ordering}"
+  fi
 
   for module in ${HADOOP_MODULES}; do
     # shellcheck disable=SC2086
@@ -373,7 +260,7 @@ function personality_modules
 
 function personality_file_tests
 {
-  local filename=$1
+  declare filename=$1
 
   yetus_debug "Using Hadoop-specific personality_file_tests"
 


Mime
View raw message