yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [2/2] yetus git commit: YETUS-523: create a separate tool for docker cleanup
Date Fri, 30 Jun 2017 18:34:43 GMT
YETUS-523: create a separate tool for docker cleanup

Signed-off-by: Sean Busbey <busbey@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/10d4d13c
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/10d4d13c
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/10d4d13c

Branch: refs/heads/master
Commit: 10d4d13cc95a814eac97a976a8de525531ac986a
Parents: eabc378
Author: Allen Wittenauer <aw@apache.org>
Authored: Fri Jun 23 08:59:40 2017 -0700
Committer: Allen Wittenauer <aw@apache.org>
Committed: Fri Jun 30 11:34:27 2017 -0700

----------------------------------------------------------------------
 .../in-progress/precommit-docker-cleanup.md     |  60 ++++++
 build.sh                                        |   7 +-
 precommit/core.d/docker.sh                      |  25 +--
 precommit/docker-cleanup.sh                     | 185 +++++++++++++++++++
 precommit/test-patch.sh                         |   2 +-
 5 files changed, 265 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md b/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md
new file mode 100644
index 0000000..9df7524
--- /dev/null
+++ b/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md
@@ -0,0 +1,60 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+docker-cleanup
+==============
+
+`docker-cleanup` is a command to perform precommit's Docker cleanup functionality outside
of patch and build testing.
+
+# Usage
+
+Executing `docker-cleanup` will peform the same Docker image as the `--robot` option.
+
+```bash
+$ docker-cleanup
+
+...
+        Removing old images
+...
+Untagged: hadoop/createrelease:3.0.0-alpha3-SNAPSHOT_10600
+Deleted: sha256:1a96c79a0a9ab538c6c7765dc908eca3e689270b778d6ae2add558e89792a7d8
+...
+                       Docker Container Maintenance
+...
+
+```
+
+
+`docker-cleanup` also supports the `--sentinel` mode to kill and remove stale running containers:
+
+```bash
+$ docker-cleanup --sentinel
+
+...
+                            Removing old images
+...
+Untagged: hadoop/createrelease:3.0.0-alpha3-SNAPSHOT_10600
+Deleted: sha256:1a96c79a0a9ab538c6c7765dc908eca3e689270b778d6ae2add558e89792a7d8
+...
+
+                       Docker Container Maintenance
+...
+Attempting to remove docker container /big_kowalevski [5ffd22a56ebcfe38d72b9078e0e7133ab6dc054115a4804e504c910bdbdeea45]
+...
+```

http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/build.sh
----------------------------------------------------------------------
diff --git a/build.sh b/build.sh
index d9fbe89..6783596 100755
--- a/build.sh
+++ b/build.sh
@@ -214,8 +214,11 @@ exec "\$(dirname -- "\${BASH_SOURCE-0}")/../lib/release-doc-maker/releasedocmake
 EOF
 chmod +x "${bin_tarball}/bin/releasedocmaker"
 
-for utility in shelldocs/shelldocs.py precommit/qbt.sh \
-               precommit/smart-apply-patch.sh precommit/test-patch.sh
+for utility in shelldocs/shelldocs.py \
+               precommit/docker-cleanup.sh \
+               precommit/qbt.sh \
+               precommit/smart-apply-patch.sh \
+               precommit/test-patch.sh
 do
   wrapper=${utility##*/}
   wrapper=${wrapper%.*}

http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/precommit/core.d/docker.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh
index 863b045..124aa0b 100755
--- a/precommit/core.d/docker.sh
+++ b/precommit/core.d/docker.sh
@@ -22,6 +22,7 @@ DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile"
 DOCKERFAIL="fallback,continue,fail"
 DOCKERSUPPORT=false
 DOCKER_ENABLE_PRIVILEGED=true
+DOCKER_CLEANUP_CMD=false
 
 declare -a DOCKER_EXTRAARGS
 
@@ -43,11 +44,15 @@ DOCKER_IMAGE_PURGE=604800
 ## @replaceable  no
 function docker_usage
 {
-  yetus_add_option "--docker" "Spawn a docker container"
+  if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
+    yetus_add_option "--docker" "Spawn a docker container"
+  fi
   yetus_add_option "--dockercmd=<file>" "Command to use as docker executable (default:
'${DOCKERCMD}')"
-  yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default:
'${DOCKERFILE_DEFAULT}')"
-  yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method
order (default: ${DOCKERFAIL})"
-  yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default:
'${DOCKER_ENABLE_PRIVILEGED}')"
+  if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
+    yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base
(default: '${DOCKERFILE_DEFAULT}')"
+    yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method
order (default: ${DOCKERFAIL})"
+    yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default:
'${DOCKER_ENABLE_PRIVILEGED}')"
+  fi
   yetus_add_option "--dockerdelrep" "In Docker mode, only report image/container deletions,
not act on them"
 }
 
@@ -129,8 +134,7 @@ function docker_initialize
   DOCKERFAIL=${DOCKERFAIL//fail/3}
   DOCKERFAIL=${DOCKERFAIL//[[:blank:]]/}
 
-  docker_exeverify
-  if [[ $? != 0 ]]; then
+  if ! docker_exeverify; then
     if [[ "${DOCKERFAIL}" =~ ^12
        || "${DOCKERFAIL}" =~ ^2 ]]; then
       add_vote_table 0 docker "Docker command '${DOCKERCMD}' not found/broken. Disabling
docker."
@@ -205,9 +209,7 @@ function docker_exeverify
     return 1
   fi
 
-  ${DOCKERCMD} info >/dev/null 2>&1
-
-  if [[ $? != 0 ]]; then
+  if ! ${DOCKERCMD} info >/dev/null 2>&1; then
     yetus_error "Docker is not functioning properly. Daemon down/unreachable?"
     return 1
   fi
@@ -487,12 +489,13 @@ function docker_version
 {
   declare vertype=$1
   declare val
-
+  declare ret
 
   # new version command
   val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null)
+  ret=$?
 
-  if [[ $? != 0 ]];then
+  if [[ ${ret} != 0 ]];then
     # old version command
     val=$(dockercmd version | ${GREP} "${vertype} version" | cut -f2 -d: | tr -d ' ')
   fi

http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/precommit/docker-cleanup.sh
----------------------------------------------------------------------
diff --git a/precommit/docker-cleanup.sh b/precommit/docker-cleanup.sh
new file mode 100755
index 0000000..4d18d21
--- /dev/null
+++ b/precommit/docker-cleanup.sh
@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no shelldocs required from this file
+# SHELLDOC-IGNORE
+
+# Make sure that bash version meets the pre-requisite
+
+if [[ -z "${BASH_VERSINFO[0]}" ]] \
+   || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
+   || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
+  echo "bash v3.2+ is required. Sorry."
+  exit 1
+fi
+
+this="${BASH_SOURCE-$0}"
+BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+BINNAME=${this##*/}
+BINNAME=${BINNAME%.sh}
+#shellcheck disable=SC2034
+STARTINGDIR=$(pwd)
+#shellcheck disable=SC2034
+USER_PARAMS=("$@")
+#shellcheck disable=SC2034
+QATESTMODE=false
+
+# dummy functions
+function add_vote_table
+{
+  true
+}
+
+function add_footer_table
+{
+  true
+}
+
+function bugsystem_finalreport
+{
+  true
+}
+
+## @description import core library routines
+## @audience private
+## @stability evolving
+function import_core
+{
+  declare filename
+
+  for filename in "${BINDIR}/core.d"/*; do
+    # shellcheck disable=SC1091
+    # shellcheck source=core.d/01-common.sh
+    . "${filename}"
+  done
+}
+
+## @description  import plugins then remove the stuff we don't need
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function import_and_clean
+{
+  importplugins
+  yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list"
+  unset BUILDTOOLS
+  unset TESTTYPES
+  unset TESTFORMATS
+
+  #shellcheck disable=SC2034
+  DOCKER_CLEANUP_CMD=true
+  #shellcheck disable=SC2034
+  DOCKERSUPPORT=true
+  #shellcheck disable=SC2034
+  ROBOT=true
+  #shellcheck disable=SC2034
+  DOCKERFAIL="fail"
+}
+
+## @description  Clean the filesystem as appropriate and then exit
+## @audience     private
+## @stability    evolving
+## @replaceable  no
+## @param        runresult
+function cleanup_and_exit
+{
+  local result=$1
+
+  if [[ ${PATCH_DIR} =~ ^/tmp/yetus
+    && -d ${PATCH_DIR} ]]; then
+    rm -rf "${PATCH_DIR}"
+  fi
+
+  # shellcheck disable=SC2086
+  exit ${result}
+}
+
+## @description  Setup the default global variables
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function setup_defaults
+{
+  common_defaults
+}
+
+## @description  Interpret the command line parameters
+## @audience     private
+## @stability    stable
+## @replaceable  no
+## @param        $@
+## @return       May exit on failure
+function parse_args
+{
+  common_args "$@"
+}
+
+## @description  Print the usage information
+## @audience     public
+## @stability    stable
+## @replaceable  no
+function yetus_usage
+{
+  import_and_clean
+
+  echo "${BINNAME} [OPTIONS]"
+
+  yetus_add_option "--debug" "If set, then output some extra stuff to stderr"
+  yetus_add_option "--sentinel" "A very aggressive robot (auto: --robot)"
+  docker_usage
+
+  yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}"
+  yetus_reset_usage
+}
+
+## @description  Large display for the user console
+## @audience     public
+## @stability    stable
+## @replaceable  no
+## @param        string
+## @return       large chunk of text
+function big_console_header
+{
+  local text="$*"
+  local spacing=$(( (75+${#text}) /2 ))
+  printf "\n\n"
+  echo "============================================================================"
+  echo "============================================================================"
+  printf "%*s\n"  ${spacing} "${text}"
+  echo "============================================================================"
+  echo "============================================================================"
+  printf "\n\n"
+}
+
+trap "cleanup_and_exit 1" HUP INT QUIT TERM
+
+import_core
+
+setup_defaults
+
+parse_args "$@"
+
+import_and_clean
+
+parse_args_plugins "$@"
+
+docker_initialize
+plugins_initialize
+
+docker_cleanup
+RESULT=$?
+
+cleanup_and_exit ${RESULT}

http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/precommit/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh
index a632f5f..75dbbe4 100755
--- a/precommit/test-patch.sh
+++ b/precommit/test-patch.sh
@@ -2994,7 +2994,7 @@ function initialize
 
   BUGLINECOMMENTS=${BUGLINECOMMENTS:-${BUGCOMMENTS}}
 
-  # we need to do this BEFORE plugins initalize
+  # we need to do this BEFORE plugins initialize
   # because they may change what they do based upon
   # docker support
   # note that docker support still isn't guaranteed


Mime
View raw message