yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject yetus git commit: YETUS-390. qbt needs an abbreviated email mode
Date Fri, 03 Jun 2016 23:35:53 GMT
Repository: yetus
Updated Branches:
  refs/heads/master ef0e1c058 -> dfd03bdc6


YETUS-390. qbt needs an abbreviated email mode

Signed-off-by: Marco Zuehlke <mzuehlke@apache.org>
Signed-off-by: Chris Nauroth <cnauroth@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/dfd03bdc
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/dfd03bdc
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/dfd03bdc

Branch: refs/heads/master
Commit: dfd03bdc68a469ab522a3d517a7e47858f97e0bb
Parents: ef0e1c0
Author: Allen Wittenauer <aw@apache.org>
Authored: Sun May 29 15:51:23 2016 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Fri Jun 3 16:25:07 2016 -0700

----------------------------------------------------------------------
 .../documentation/in-progress/precommit-qbt.md  |   7 +-
 precommit/core.d/builtin-bugsystem.sh           |   4 +-
 precommit/test-patch.d/briefreport.sh           | 254 +++++++++++++++++++
 precommit/test-patch.d/htmlout.sh               |   9 +-
 precommit/test-patch.d/jira.sh                  |   7 +-
 precommit/test-patch.sh                         |  72 +++---
 6 files changed, 308 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/asf-site-src/source/documentation/in-progress/precommit-qbt.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/in-progress/precommit-qbt.md b/asf-site-src/source/documentation/in-progress/precommit-qbt.md
index 3e8f578..badaf60 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-qbt.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-qbt.md
@@ -61,4 +61,9 @@ ${FILE,path="<report-file-path>"}
 </body></html>
 ```
 
-NOTE: Be aware that ASF mailing lists do not allow HTML formatted email.
+If your mailing lists do not allow HTML-formatted email, then the `--brief-report-file`
+provides a solution.  This option creates a very plain, reduced content text file
+suitable for email.  It contains just the barebones information needed to get
+information on failures: what voted -1, what tests failed, what subsystems are long
+running (configurable with the `--brief-report-long` opton), and a list of any
+attached log files.

http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/core.d/builtin-bugsystem.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/builtin-bugsystem.sh b/precommit/core.d/builtin-bugsystem.sh
index c7d9a6c..cc14572 100755
--- a/precommit/core.d/builtin-bugsystem.sh
+++ b/precommit/core.d/builtin-bugsystem.sh
@@ -44,6 +44,7 @@ function console_finalreport
   declare line
   declare seccoladj=0
   declare spcfx=${PATCH_DIR}/spcl.txt
+  declare calctime
 
   if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then
     exec 6>&1
@@ -114,6 +115,7 @@ function console_finalreport
     vote=$(echo "${ourstring}" | cut -f2 -d\|)
     subs=$(echo "${ourstring}"  | cut -f3 -d\|)
     ela=$(echo "${ourstring}" | cut -f4 -d\|)
+    calctime=$(clock_display "${ela}")
     comment=$(echo "${ourstring}"  | cut -f5 -d\|)
 
     echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}"
@@ -121,7 +123,7 @@ function console_finalreport
     ${SED} -e '1d' "${commentfile1}"  > "${commentfile2}"
 
     printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \
-      "${subs}" "${ela}" "${normaltop}"
+      "${subs}" "${calctime}" "${normaltop}"
     while read -r line; do
       printf "|      | %*s |            | %-s\n" ${seccoladj} " " "${line}"
     done < "${commentfile2}"

http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/test-patch.d/briefreport.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.d/briefreport.sh b/precommit/test-patch.d/briefreport.sh
new file mode 100755
index 0000000..144e234
--- /dev/null
+++ b/precommit/test-patch.d/briefreport.sh
@@ -0,0 +1,254 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_bugsystem brieftext
+
+BRIEFOUT_LONGRUNNING=3600
+
+## @description  Usage info for brieftext plugin
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+function brieftext_usage
+{
+  yetus_add_option "--brief-report-file=<file>" "Save a very brief, plain text report
to a file"
+  yetus_add_option "--brief-report-long=<seconds>" "Time in seconds to use as long
running subsystem threshold (Default: ${BRIEFOUT_LONGRUNNING})"
+
+}
+
+## @description  Option parsing for brieftext plugin
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+function brieftext_parse_args
+{
+  declare i
+  declare fn
+
+  for i in "$@"; do
+    case ${i} in
+      --brief-report-file=*)
+        fn=${i#*=}
+      ;;
+      --brief-report-long=*)
+        BRIEFOUT_LONGRUNNING=${i#*=}
+      ;;
+    esac
+  done
+
+  if [[ -n "${fn}" ]]; then
+    touch "${fn}" 2>/dev/null
+    if [[ $? != 0 ]]; then
+      yetus_error "WARNING: cannot create ${fn}. Ignoring."
+    else
+      BRIEFOUT_REPORTFILE=$(yetus_abs "${fn}")
+    fi
+  fi
+}
+
+## @description  Give access to the brief report file in docker mode
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+function brieftext_docker_support
+{
+  if [[ -n ${BRIEFOUT_REPORTFILE} ]]; then
+    DOCKER_EXTRAARGS=("${DOCKER_EXTRAARGS[@]}" "-v" "${BRIEFOUT_REPORTFILE}:${BRIEFOUT_REPORTFILE}")
+  fi
+}
+
+## @description  Only print selected information to a report file
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+## @param        runresult
+## @return       0 on success
+## @return       1 on failure
+function brieftext_finalreport
+{
+  declare result=$1
+  shift
+  declare i=0
+  declare ourstring
+  declare vote
+  declare subs
+  declare ela
+  declare version
+  declare -a failed
+  declare -a long
+  declare -a filtered
+  declare hours
+  declare newtime
+  declare havelogs=false
+
+  if [[ -z "${BRIEFOUT_REPORTFILE}" ]]; then
+    return
+  fi
+
+  big_console_header "Writing Brief Report to ${BRIEFOUT_REPORTFILE}"
+
+
+  if [[ ${result} == 0 ]]; then
+    printf "\n\n+1 overall\n\n" > "${BRIEFOUT_REPORTFILE}"
+  else
+    printf "\n\n-1 overall\n\n" > "${BRIEFOUT_REPORTFILE}"
+  fi
+
+  i=0
+  until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do
+    ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ')
+    vote=$(echo "${ourstring}" | cut -f2 -d\|)
+    subs=$(echo "${ourstring}"  | cut -f3 -d\|)
+    ela=$(echo "${ourstring}" | cut -f4 -d\|)
+
+    if [[ ${vote// } = -1 ]]; then
+      failed=("${failed[@]}" "${subs}")
+    fi
+
+    if [[ ${vote// } = -0 ]]; then
+      filtered=("${filtered[@]}" "${subs}")
+    fi
+
+    if [[ ${ela// } -gt ${BRIEFOUT_LONGRUNNING} ]]; then
+      long=("${long[@]}" "${subs}")
+    fi
+
+    ((i=i+1))
+  done
+
+  tmparray=($(printf "%s\n" "${failed[@]}" | sort -u))
+  failed=("${tmparray[@]}")
+  tmparray=($(printf "%s\n" "${filtered[@]}" | sort -u))
+  filtered=("${tmparray[@]}")
+  tmparray=($(printf "%s\n" "${long[@]}" | sort -u))
+  long=("${tmparray[@]}")
+
+  if [[ ${#failed[@]} -gt 0 ]]; then
+    {
+      echo ""
+      echo "The following subsystems voted -1:"
+      echo "    ${failed[*]}"
+      echo ""
+    } >> "${BRIEFOUT_REPORTFILE}"
+  fi
+
+  if [[ ${#filtered[@]} -gt 0 ]]; then
+    {
+      echo ""
+      echo "The following subsystems voted -1 but"
+      echo "were configured to be filtered/ignored:"
+      echo "    ${filtered[*]}"
+      echo ""
+    } >> "${BRIEFOUT_REPORTFILE}"
+  fi
+
+  if [[ ${#long[@]} -gt 0 ]]; then
+    {
+      echo ""
+      echo "The following subsystems are considered long running:"
+      printf "(runtime bigger than "
+      # We would use clock_display here, but we don't have the
+      # restrictions that the vote_table has on size plus
+      # we're almost certainly going to be measured in hours
+      if [[ ${BRIEFOUT_LONGRUNNING} -ge 3600 ]]; then
+        hours=$((BRIEFOUT_LONGRUNNING/3600))
+        newtime=$((BRIEFOUT_LONGRUNNING-hours*3600))
+        printf "%sh %02sm %02ss" ${hours} $((newtime/60)) $((newtime%60))
+      else
+        printf "%sm %02ss" $((BRIEFOUT_LONGRUNNING/60)) $((BRIEFOUT_LONGRUNNING%60))
+      fi
+      echo ")"
+      echo "    ${long[*]}"
+      echo ""
+    } >> "${BRIEFOUT_REPORTFILE}"
+  fi
+
+  if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then
+    {
+      echo ""
+      echo "Specific tests:"
+    } >> "${BRIEFOUT_REPORTFILE}"
+
+    i=0
+    until [[ $i -gt ${#TP_TEST_TABLE[@]} ]]; do
+      ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ')
+      vote=$(echo "${ourstring}" | cut -f2 -d\|)
+      subs=$(echo "${ourstring}"  | cut -f3 -d\|)
+      {
+        if [[ -n "${vote// }" ]]; then
+          echo ""
+          printf "   %s:\n" "${vote}"
+          echo ""
+          vote=""
+        fi
+        printf "      %s\n" "${subs}"
+      } >> "${BRIEFOUT_REPORTFILE}"
+      ((i=i+1))
+    done
+  fi
+
+  if [[ -f "${BINDIR}/../VERSION" ]]; then
+    version=$(cat "${BINDIR}/../VERSION")
+  elif [[ -f "${BINDIR}/VERSION" ]]; then
+    version=$(cat "${BINDIR}/VERSION")
+  fi
+
+  i=0
+  until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
+    if [[ "${TP_FOOTER_TABLE[${i}]}" =~ \@\@BASE\@\@ ]]; then
+      havelogs=true
+      break
+    fi
+    ((i=i+1))
+  done
+
+  if [[ "${havelogs}" == true ]]; then
+    vote=""
+    until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
+      if [[ "${TP_FOOTER_TABLE[${i}]}" =~ \@\@BASE\@\@ ]]; then
+        ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ')
+        subs=$(echo "${TP_FOOTER_TABLE[${i}]}" | cut -f2 -d\|)
+        comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
+                    cut -f3 -d\| |
+                    ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
+        # shellcheck disable=SC2016
+        size=$(du -sh "${comment// }" | ${AWK} '{print $1}')
+        if [[ -n "${BUILD_URL}" ]]; then
+          comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
+                    cut -f3 -d\| |
+                    ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
+        fi
+        {
+          if [[ "${subs}" != "${vote}" ]]; then
+            echo ""
+            printf "   %s:\n" "${subs// }"
+            echo ""
+            vote=${subs}
+          fi
+          printf "      %s [%s]\n" "${comment}" "${size}"
+        } >> "${BRIEFOUT_REPORTFILE}"
+      fi
+      ((i=i+1))
+    done
+  fi
+
+  {
+   echo ""
+   echo "Powered by" "Apache Yetus ${version}   http://yetus.apache.org"
+   echo ""
+  } >> "${BRIEFOUT_REPORTFILE}"
+
+}

http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/test-patch.d/htmlout.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.d/htmlout.sh b/precommit/test-patch.d/htmlout.sh
index bf6d06e..f2dab40 100755
--- a/precommit/test-patch.d/htmlout.sh
+++ b/precommit/test-patch.d/htmlout.sh
@@ -81,6 +81,7 @@ function htmlout_finalreport
   declare subs
   declare color
   declare comment
+  declare calctime
 
   rm "${commentfile}" 2>/dev/null
 
@@ -98,7 +99,7 @@ function htmlout_finalreport
     else
       echo "<tr><th><font color=\"red\">-1 overall</font></th></tr>"
     fi
-    echo "</table></tbody>"
+    echo "</tbody></table>"
     echo "<p></p>"
   } >  "${commentfile}"
 
@@ -126,6 +127,7 @@ function htmlout_finalreport
     vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ')
     subs=$(echo "${ourstring}"  | cut -f3 -d\|)
     ela=$(echo "${ourstring}" | cut -f4 -d\|)
+    calctime=$(clock_display "${ela}")
     comment=$(echo "${ourstring}"  | cut -f5 -d\|)
 
     # summary line
@@ -147,6 +149,9 @@ function htmlout_finalreport
         0)
           color="blue"
         ;;
+        -0)
+          color="orange"
+        ;;
         *)
           color="black"
         ;;
@@ -157,7 +162,7 @@ function htmlout_finalreport
       echo "<tr>"
       printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${vote}"
       printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${subs}"
-      printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${ela}"
+      printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${calctime}"
       printf "<td><font color=\"%s\">%s</font></td>" "${color}" "${comment}"
       echo "</tr>"
     } >> "${commentfile}"

http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/test-patch.d/jira.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.d/jira.sh b/precommit/test-patch.d/jira.sh
index 60afe00..7da9280 100755
--- a/precommit/test-patch.d/jira.sh
+++ b/precommit/test-patch.d/jira.sh
@@ -357,6 +357,7 @@ function jira_finalreport
   declare subs
   declare color
   declare comment
+  declare calctime
 
   rm "${commentfile}" 2>/dev/null
 
@@ -395,6 +396,7 @@ function jira_finalreport
     vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ')
     subs=$(echo "${ourstring}"  | cut -f3 -d\|)
     ela=$(echo "${ourstring}" | cut -f4 -d\|)
+    calctime=$(clock_display "${ela}")
     comment=$(echo "${ourstring}"  | cut -f5 -d\|)
 
     # summary line
@@ -416,6 +418,9 @@ function jira_finalreport
         0)
           color="blue"
         ;;
+        -0)
+          color="orange"
+        ;;
         *)
           color="black"
         ;;
@@ -425,7 +430,7 @@ function jira_finalreport
     printf "| {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color} | {color:%s}%s{color}
|\n" \
       "${color}" "${vote}" \
       "${color}" "${subs}" \
-      "${color}" "${ela}" \
+      "${color}" "${calctime}" \
       "${color}" "${comment}" \
       >> "${commentfile}"
     ((i=i+1))

http://git-wip-us.apache.org/repos/asf/yetus/blob/dfd03bdc/precommit/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh
index ec18134..a818af0 100755
--- a/precommit/test-patch.sh
+++ b/precommit/test-patch.sh
@@ -220,52 +220,41 @@ function add_header_line
 
 ## @description  Add to the output table. If the first parameter is a number
 ## @description  that is the vote for that column and calculates the elapsed time
-## @description  based upon the last start_clock().  If it the string null, then it is
-## @description  a special entry that signifies extra
-## @description  content for the final column.  The second parameter is the reporting
+## @description  based upon the last start_clock().  The second parameter is the reporting
 ## @description  subsystem (or test) that is providing the vote.  The second parameter
 ## @description  is always required.  The third parameter is any extra verbage that goes
 ## @description  with that subsystem.
 ## @audience     public
 ## @stability    stable
 ## @replaceable  no
-## @param        +1/0/-1/null
+## @param        +1/0/-1
 ## @param        subsystem
 ## @param        string
-## @return       Elapsed time display
 function add_vote_table
 {
   declare value=$1
   declare subsystem=$2
   shift 2
 
-  declare calctime
   # apparently shellcheck doesn't know about declare -r
   #shellcheck disable=SC2155
   declare -r elapsed=$(stop_clock)
   declare filt
 
-  yetus_debug "add_vote_table ${value} ${subsystem} ${*}"
-
-  calctime=$(clock_display "${elapsed}")
+  yetus_debug "add_vote_table ${value} ${subsystem} ${elapsed} ${*}"
 
   if [[ ${value} == "1" ]]; then
     value="+1"
   fi
 
   for filt in "${VOTE_FILTER[@]}"; do
-    if [[ "${subsystem}" = "${filt}" ]]; then
-      value=0
+    if [[ "${subsystem}" == "${filt}" && "${value}" == -1 ]]; then
+      value=-0
     fi
   done
 
-  if [[ -z ${value} ]]; then
-    # shellcheck disable=SC2034
-    TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="|  | ${subsystem} | | ${*:-} |"
-  else
-    # shellcheck disable=SC2034
-    TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${calctime} | $* |"
-  fi
+  # shellcheck disable=SC2034
+  TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| ${value} | ${subsystem} | ${elapsed} | $* |"
   ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1))
 
   if [[ "${value}" = -1 ]]; then
@@ -358,7 +347,7 @@ function finish_vote_table
   echo ""
 
   # shellcheck disable=SC2034
-  TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | | ${calctime} | |"
+  TP_VOTE_TABLE[${TP_VOTE_COUNTER}]="| | | ${elapsed} | |"
   ((TP_VOTE_COUNTER=TP_VOTE_COUNTER+1 ))
 }
 
@@ -1863,29 +1852,40 @@ function modules_messages
   TIMER=${oldtimer}
 }
 
-## @description  Add a test result
+## @description  Add or update a test result. Update requires
+## @description  at least the first two parameters.
+## @description  WARNING: If the message is updated,
+## @description  then the JDK version is also calculated to match
+## @description  the current JAVA_HOME.
 ## @audience     public
 ## @stability    evolving
 ## @replaceable  no
-## @param        module
-## @param        runtime
+## @param        moduleindex
+## @param        -1-0|0|+1
+## @param        logvalue
+## @param        message
 function module_status
 {
-  local index=$1
-  local value=$2
-  local log=$3
-  shift 3
+  declare index=$1
+  declare value=$2
+  shift 2
+  declare log=$1
+  shift
 
-  local jdk
+  declare jdk
 
   jdk=$(report_jvm_version "${JAVA_HOME}")
 
   if [[ -n ${index}
     && ${index} =~ ^[0-9]+$ ]]; then
     MODULE_STATUS[${index}]="${value}"
-    MODULE_STATUS_LOG[${index}]="${log}"
-    MODULE_STATUS_JDK[${index}]=" with JDK v${jdk}"
-    MODULE_STATUS_MSG[${index}]="${*}"
+    if [[ -n ${log} ]]; then
+      MODULE_STATUS_LOG[${index}]="${log}"
+    fi
+    if [[ -n $1 ]]; then
+      MODULE_STATUS_JDK[${index}]=" with JDK v${jdk}"
+      MODULE_STATUS_MSG[${index}]="${*}"
+    fi
   else
     yetus_error "ASSERT: module_status given bad index: ${index}"
     yetus_error "ASSERT: module_stats $*"
@@ -2097,7 +2097,6 @@ function check_unittests
   declare statusjdk
   declare formatresult=0
   declare needlog
-  declare unitlogs
 
   if ! verify_needed_test unit; then
     return 0
@@ -2127,11 +2126,6 @@ function check_unittests
 
     ((result=result+$?))
 
-    modules_messages patch unit false
-    if [[ ${result} == 0 ]]; then
-      continue
-    fi
-
     i=0
     until [[ $i -eq ${#MODULE[@]} ]]; do
       module=${MODULE[${i}]}
@@ -2155,7 +2149,7 @@ function check_unittests
       done
 
       if [[ ${needlog} == 1 ]]; then
-        unitlogs="${unitlogs} @@BASE@@/patch-unit-${fn}.txt"
+        module_status ${i} -1 "patch-unit-${fn}.txt"
       fi
 
       popd >/dev/null
@@ -2173,9 +2167,7 @@ function check_unittests
   done
   JAVA_HOME=${savejavahome}
 
-  if [[ -n "${unitlogs}" ]]; then
-    add_footer_table "unit test logs" "${unitlogs}"
-  fi
+  modules_messages patch unit false
 
   if [[ ${JENKINS} == true ]]; then
     add_footer_table "${statusjdk} Test Results" "${BUILD_URL}testReport/"


Mime
View raw message