yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject yetus git commit: YETUS-280. build systems should be able to predetermine module order
Date Wed, 13 Jan 2016 16:57:51 GMT
Repository: yetus
Updated Branches:
  refs/heads/master 6ce890bcd -> e330e3fdf


YETUS-280. build systems should be able to predetermine module order

Signed-off-by: Sean Busbey <busbey@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/e330e3fd
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/e330e3fd
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/e330e3fd

Branch: refs/heads/master
Commit: e330e3fdf111ce0ac22076db59cc2f34780a347c
Parents: 6ce890b
Author: Allen Wittenauer <aw@apache.org>
Authored: Tue Jan 12 14:06:37 2016 -0800
Committer: Allen Wittenauer <aw@apache.org>
Committed: Wed Jan 13 08:57:31 2016 -0800

----------------------------------------------------------------------
 precommit/personality/hadoop.sh |   8 ---
 precommit/test-patch.d/maven.sh | 130 +++++++++++++++++++++++++++++++++++
 precommit/test-patch.sh         |   8 ++-
 3 files changed, 137 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/yetus/blob/e330e3fd/precommit/personality/hadoop.sh
----------------------------------------------------------------------
diff --git a/precommit/personality/hadoop.sh b/precommit/personality/hadoop.sh
index 0656491..0ccf137 100755
--- a/precommit/personality/hadoop.sh
+++ b/precommit/personality/hadoop.sh
@@ -63,15 +63,7 @@ function hadoop_module_manipulation
     fi
   done
 
-  # ${startingmodules} is already sorted and uniq'd.
-  # let's remove child modules if we're going to
-  # touch their parent.
   passed_modules=${startingmodules}
-  for module in ${startingmodules}; do
-    yetus_debug "Stripping ${module}"
-    # shellcheck disable=SC2086
-    passed_modules=$(echo ${passed_modules} | tr ' ' '\n' | ${GREP} -v ${module}/ )
-  done
 
   yetus_debug "hmm pre-ordering: ${startingmodules}"
 

http://git-wip-us.apache.org/repos/asf/yetus/blob/e330e3fd/precommit/test-patch.d/maven.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.d/maven.sh b/precommit/test-patch.d/maven.sh
index 7068ea6..861c31c 100755
--- a/precommit/test-patch.d/maven.sh
+++ b/precommit/test-patch.d/maven.sh
@@ -15,6 +15,8 @@
 # limitations under the License.
 
 declare -a MAVEN_ARGS=("--batch-mode")
+declare -a MAVEN_ALREADY=()
+declare -a MAVEN_NEWLIST=()
 
 if [[ -z "${MAVEN_HOME:-}" ]]; then
   MAVEN=mvn
@@ -24,6 +26,7 @@ fi
 
 MAVEN_CUSTOM_REPOS=false
 MAVEN_CUSTOM_REPOS_DIR="${HOME}/yetus-m2"
+MAVEN_DEPENDENCY_ORDER=true
 
 add_test_type mvnsite
 add_test_type mvneclipse
@@ -34,6 +37,7 @@ function maven_usage
   yetus_add_option "--mvn-cmd=<cmd>" "The 'mvn' command to use (default \${MAVEN_HOME}/bin/mvn,
or 'mvn')"
   yetus_add_option "--mvn-custom-repos" "Use per-project maven repos"
   yetus_add_option "--mvn-custom-repos-dir=dir" "Location of repos, default is '${MAVEN_CUSTOM_REPOS_DIR}'"
+  yetus_add_option "--mvn-deps-order=<bool>" "Disable maven's auto-dependency module
ordering (Default: '${MAVEN_DEPENDENCY_ORDER}')"
   yetus_add_option "--mvn-settings=file" "File to use for settings.xml"
 }
 
@@ -52,6 +56,9 @@ function maven_parse_args
       --mvn-custom-repos-dir=*)
         MAVEN_CUSTOM_REPOS_DIR=${i#*=}
       ;;
+      --mvn-deps-order=*)
+        MAVEN_DEPENDENCY_ORDER=${i#*=}
+      ;;
       --mvn-settings=*)
         MAVEN_SETTINGS=${i#*=}
         if [[ -f ${MAVEN_SETTINGS} ]]; then
@@ -524,3 +531,126 @@ function maven_docker_support
       >> "${PATCH_DIR}/buildtool-docker-params.txt"
   fi
 }
+
+## @description  worker for maven reordering
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+## @param        repostatus
+## @param        module
+function maven_reorder_module_process
+{
+  declare repostatus=$1
+  declare module=$2
+  declare indexd
+  declare indexm
+  declare deplist
+  declare fn
+  declare basemod
+
+  yetus_debug "Maven: check order: ${module}"
+
+  # this is a recursive function; no need to process
+  # modules that we have already seen
+  if [[ " ${MAVEN_ALREADY[@]} " =~ \ ${module}\  ]]; then
+    return
+  fi
+
+  # immediately place a module in the list to avoid
+  # loops
+  MAVEN_ALREADY=("${MAVEN_ALREADY[@]}" "${module}")
+
+  pushd "${BASEDIR}/${module}" >/dev/null
+  fn=$(module_file_fragment "${module}")
+
+  # get the dependency list from maven
+  #shellcheck disable=SC2046
+  echo_and_redirect "${PATCH_DIR}/maven-${repostatus}-dependencylist-${fn}.txt" \
+    $("${BUILDTOOL}_executor") dependency:list \
+      -DoutputFile="${PATCH_DIR}/${repostatus}-mvndeporder-${fn}.txt"
+
+  # if this fails, there's really not much we can do other than tell the
+  # user that it did. So we'll press on and just hope for the best
+
+  if [[ $? != 0 ]]; then
+    add_vote_table -1 mvndep "${repostatus}'s ${module} dependency:list failed"
+    add_footer_table mvndep "${fn}: @@BASE@@/maven-${repostatus}-dependencylist-${fn}.txt"
+  fi
+
+  # take the list from maven, and pull out the artifactids. this means
+  # we have an assumption that the module names we are given map
+  # 1:1 to the artifactid
+  deplist=$(cut -f2 -d: "${PATCH_DIR}/${repostatus}-mvndeporder-${fn}.txt")
+
+  # loop through the generated dependency artificatid list
+  # and try to match it to any of the other modules we are working on
+  # if it is a match, process it.  dependencies that aren't being hit
+  # by the patch should be irrelevant since maven will pick them up
+  # from the repo caches.  if a dependency really does matter, then
+  # the personality will have a chance to manipulate this list later
+  for indexd in ${deplist}; do
+    for indexm in ${CHANGED_MODULES}; do
+
+      # modules could be foo/bar, where bar is the artifactid
+      # so get the basename and compare that too
+      basemod=$(basename "${indexm}")
+      if [[ " ${indexd} " = " ${indexm} "
+         || " ${indexd} " = " ${basemod} " ]]; then
+        maven_reorder_module_process "${repostatus}" "${indexm}"
+      fi
+    done
+  done
+  popd >/dev/null
+
+  # at this point, any modules that we care about should have been placed
+  # before us.  now add the one we've been working on.  if the artificatids
+  # don't match, then we're effectively back to the old behavior of
+  # using fs globs and the personality will have to do the hard work
+  yetus_debug "Maven: placing ${module}"
+  MAVEN_NEWLIST=("${MAVEN_NEWLIST[@]}" " ${module} ")
+}
+
+## @description  take a stab at reordering modules based upon
+## @description  maven dependency order
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+## @param        repostatus
+## @param        module
+function maven_reorder_modules
+{
+  declare repostatus=$1
+  declare index
+
+  # sidenote: this code does not get called when there is only one
+  # module, so no reason to shortcut that
+
+  if [[ "${MAVEN_DEPENDENCY_ORDER}" != "true" ]]; then
+    return
+  fi
+
+  big_console_header "Determining Maven Dependency Order"
+
+  start_clock
+
+  # this will get called again, so always make sure we start fresh
+  MAVEN_ALREADY=()
+  MAVEN_NEWLIST=()
+
+  yetus_debug "Maven: start re-ordering modules"
+  yetus_debug "Starting list: ${CHANGED_MODULES}"
+
+  # sort the list.  parents will likely get us most of the
+  # dependency list sooner/more correct, esp if . is in the list
+  # shellcheck disable=SC2086
+  for index in $(echo ${CHANGED_MODULES} | tr ' ' '\n' |  sort); do
+    maven_reorder_module_process "${repostatus}" "${index}"
+  done
+
+  CHANGED_MODULES="${MAVEN_NEWLIST[*]}"
+
+  yetus_debug "Maven: finish re-ordering modules"
+  yetus_debug "Finished list: ${CHANGED_MODULES}"
+
+  add_vote_table 0 mvndep "Maven dependency ordering for ${repostatus}"
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/yetus/blob/e330e3fd/precommit/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh
index f2f9c59..a71c299 100755
--- a/precommit/test-patch.sh
+++ b/precommit/test-patch.sh
@@ -1026,9 +1026,11 @@ function module_skipdir
 ## @audience     private
 ## @stability    stable
 ## @replaceable  no
+## @param        repostatus
 ## @return       None; sets ${CHANGED_MODULES}
 function find_changed_modules
 {
+  local repostatus=$1
   local i
   local changed_dirs
   local builddirs
@@ -1115,6 +1117,10 @@ function find_changed_modules
 
   #shellcheck disable=SC2034
   CHANGED_UNION_MODULES="${builddir}"
+
+  if declare -f ${BUILDTOOL}_reorder_modules >/dev/null; then
+    "${BUILDTOOL}_reorder_modules" "${repostatus}"
+  fi
 }
 
 ## @description  git checkout the appropriate branch to test.  Additionally, this calls
@@ -2506,7 +2512,7 @@ function compile_cycle
   declare result=0
   declare plugin
 
-  find_changed_modules
+  find_changed_modules "${codebase}"
 
   for plugin in ${PROJECT_NAME} ${BUILDTOOL} ${TESTTYPES} ${TESTFORMATS}; do
     if declare -f ${plugin}_precompile >/dev/null 2>&1; then


Mime
View raw message