yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [15/17] yetus git commit: YETUS-15. build environment
Date Sun, 11 Nov 2018 22:19:31 GMT
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/build.sh
----------------------------------------------------------------------
diff --git a/build.sh b/build.sh
deleted file mode 100755
index a6f2dc9..0000000
--- a/build.sh
+++ /dev/null
@@ -1,283 +0,0 @@
-#!/bin/bash -e
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Temporary script for building tarballs. See YETUS-125 to help
-# create a more sustainable build system.
-#
-# Pass --release to get release checks
-# Pass --deploy to deploy maven snapshot artifacts
-#
-# Presumes you have
-#   * maven 3.2.0+
-#   * jdk 1.7+ (1.7 in --release)
-#   * ruby + gems needed to run middleman
-#   * python + python-dateutil
-
-## @description  Verify that all required dependencies exist
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        true iff this is a release build
-## @return       1 - Some dependencies are missing
-## @return       0 - All dependencies exist
-function detect_dependencies
-{
-  declare is_release=$1
-  local exit_code=0
-  if ! [ -x "$(command -v java)" ]; then
-    echo "Java not found! Must install JDK version >= 1.7" >&2
-    exit_code=1
-  fi
-  if ! [ -x "$(command -v mvn)" ]; then
-    echo "Apache Maven not found! Must install version >= 3.2.0" >&2
-    echo "Download it at https://maven.apache.org/download.cgi" >&2
-    exit_code=1
-  fi
-  if ! [ -x "$(command -v bundle)" ]; then
-    echo "building docs requires a Ruby executable bundle." >&2
-    echo "Install it by executing 'gem install bundler && bundle install'" >&2
-    exit_code=1
-  fi
-
-  ! python -c 'import dateutil.parser' 2>/dev/null
-  if [ "$?" -eq "0" ]; then
-    echo "Building release docs requires the python-dateutil module" >&2
-    echo "Install it by executing 'pip install python-dateutil'" >&2
-    exit_code=1
-  fi
-
-  if ! [ -x "$(command -v tar)" ]; then
-    echo "Building archives requires the 'tar' command." >&2
-    exit_code=1
-  fi
-
-  if [ "${is_release}" = "true" ] && ! [ -x "$(command -v pax)" ]; then
-    echo "building the release source archive requires the 'pax' command." >&2
-    exit_code=1
-  fi
-
-  if [[ "${exit_code}" -ne "0" ]]; then
-    echo "Some dependencies are missing. Exit now." >&2
-  fi
-  return ${exit_code}
-}
-
-YETUS_VERSION=$(cat VERSION)
-RAT_DOWNLOAD_URL=https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.11/apache-rat-0.11.jar
-
-release=false
-offline=false
-for arg in "$@"; do
-  case ${arg} in
-    --release)
-      release=true
-      ;;
-    --offline)
-      offline=true
-      ;;
-    --deploy)
-      deploy=true
-      ;;
-  esac
-done
-
-echo "working on version '${YETUS_VERSION}'"
-
-detect_dependencies "${release}"
-mkdir -p target
-
-if [ "${offline}" != "true" ]; then
-  JIRA_VERSION="${YETUS_VERSION%%-SNAPSHOT}"
-  echo "generating release docs."
-  # Note that we use the bare python here instead of the wrapper script, since we
-  # create said script.
-  release-doc-maker/releasedocmaker.py --lint=all --license --outputdir target \
-                                                   --project YETUS "--version=${JIRA_VERSION}" \
-                                                   --projecttitle="Apache Yetus" --usetoday
-else
-  echo "in offline mode, skipping release notes."
-fi
-
-MAVEN_ARGS=()
-if [ "${offline}" = "true" ]; then
-  MAVEN_ARGS=("${MAVEN_ARGS[@]}" --offline)
-fi
-
-if [ "${release}" = "true" ]; then
-  MAVEN_ARGS=("${MAVEN_ARGS[@]}" -Papache-release)
-  echo "hard reseting working directory."
-  git reset --hard HEAD
-
-  if [ ! -f target/rat.jar ]; then
-    if [ "${offline}" != "true" ]; then
-      echo "downloading rat jar file to '$(pwd)/target/'"
-      curl -o target/rat.jar "${RAT_DOWNLOAD_URL}"
-    else
-      echo "in offline mode, can't retrieve rat jar. will skip license check."
-    fi
-  fi
-  echo "creating source tarball at '$(pwd)/target/'"
-  rm "target/yetus-${YETUS_VERSION}-src".tar* 2>/dev/null || true
-  pax -w -f "target/yetus-${YETUS_VERSION}-src.tar" -s "/target/yetus-${YETUS_VERSION}/" target/RELEASENOTES.md target/CHANGELOG.md
-  current=$(basename "$(pwd)")
-  #shellcheck disable=SC2038
-  (cd ..; find "${current}" \( -name target -o -name publish -o -name .git \) -prune -o ! -type d -print | xargs pax -w -a -f "${current}/target/yetus-${YETUS_VERSION}-src.tar" -s "/${current}/yetus-${YETUS_VERSION}/")
-  gzip "target/yetus-${YETUS_VERSION}-src.tar"
-fi
-
-echo "running maven builds for java components"
-# build java components
-if [[ "${deploy}" = true ]]; then
-  mvncmd=deploy
-else
-  mvncmd=install
-fi
-mvn "${MAVEN_ARGS[@]}" "${mvncmd}" --file yetus-project/pom.xml
-mvn "${MAVEN_ARGS[@]}" -Pinclude-jdiff-module "${mvncmd}" javadoc:aggregate --file audience-annotations-component/pom.xml
-
-echo "building documentation"
-# build docs after javadocs
-docs_out=$(cd asf-site-src && bundle exec middleman build)
-echo "${docs_out}"
-
-bin_tarball="target/bin-dir/yetus-${YETUS_VERSION}"
-echo "creating staging area for convenience binary at '$(pwd)/${bin_tarball}'"
-rm -rf "${bin_tarball}" 2>/dev/null || true
-mkdir -p "${bin_tarball}"
-
-for i in LICENSE NOTICE; do
-  lines=$(grep -n 'Apache Yetus Source' "${i}" | cut -f1 -d:)
-  if [[ -z "${lines}" ]]; then
-    cp -p "${i}" "${bin_tarball}"
-  else
-    ((lines=lines-2))
-    head -n "${lines}" "${i}" > "${bin_tarball}/${i}"
-  fi
-done
-
-cp target/RELEASENOTES.md target/CHANGELOG.md "${bin_tarball}"
-cp -r asf-site-src/publish/documentation/in-progress "${bin_tarball}/docs"
-
-mkdir -p "${bin_tarball}/lib"
-cp VERSION "${bin_tarball}/lib/"
-
-mkdir -p "${bin_tarball}/lib/yetus-project"
-cp yetus-project/pom.xml "${bin_tarball}/lib/yetus-project/yetus-project-${YETUS_VERSION}.pom"
-
-mkdir -p "${bin_tarball}/lib/audience-annotations"
-cp audience-annotations-component/audience-annotations/target/audience-annotations-*.jar \
-   audience-annotations-component/audience-annotations-jdiff/target/audience-annotations-jdiff-*.jar \
-   "${bin_tarball}/lib/audience-annotations/"
-
-cp -r shelldocs "${bin_tarball}/lib/"
-
-cp -r release-doc-maker "${bin_tarball}/lib/"
-
-cp -r precommit "${bin_tarball}/lib/"
-ln -s test-patch.sh "${bin_tarball}/lib/precommit/qbt.sh"
-
-mkdir -p "${bin_tarball}/bin"
-
-# Make a special version of the shell wrapper for releasedocmaker
-# that maintains the ability to have '--lint' mean '--lint=all'
-cat >"${bin_tarball}/bin/releasedocmaker" <<EOF
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-args=()
-for arg in "\${@}"; do
-  if [ "\${arg}" = "-n" ] || [ "\${arg}" = "--lint" ]; then
-    args=("\${args[@]}" "--lint=all")
-  else
-    args=("\${args[@]}" "\${arg}")
-  fi
-done
-
-exec "\$(dirname -- "\${BASH_SOURCE-0}")/../lib/release-doc-maker/releasedocmaker.py" "\${args[@]}"
-EOF
-chmod +x "${bin_tarball}/bin/releasedocmaker"
-
-for utility in shelldocs/shelldocs.py \
-               precommit/docker-cleanup.sh \
-               precommit/qbt.sh \
-               precommit/smart-apply-patch.sh \
-               precommit/test-patch.sh \
-               precommit/jenkins/jenkins-admin.py
-do
-  wrapper=${utility##*/}
-  wrapper=${wrapper%.*}
-  cat >"${bin_tarball}/bin/${wrapper}" <<EOF
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-exec "\$(dirname -- "\${BASH_SOURCE-0}")/../lib/${utility}" "\${@}"
-EOF
-  chmod +x "${bin_tarball}/bin/${wrapper}"
-done
-
-bin_file="target/yetus-${YETUS_VERSION}-bin.tar.gz"
-echo "creating convenience binary in '$(pwd)/target'"
-rm "${bin_file}" 2>/dev/null || true
-tar -C "$(dirname "${bin_tarball}")" -czf "${bin_file}" "$(basename "${bin_tarball}")"
-
-if [ "${release}" = "true" ] && [ -f target/rat.jar ]; then
-  echo "checking asf licensing requirements for source tarball '$(pwd)/target/yetus-${YETUS_VERSION}-src.tar.gz'."
-  rm -rf target/source-unpack 2>/dev/null || true
-  mkdir target/source-unpack
-  tar -C target/source-unpack -xzf "target/yetus-${YETUS_VERSION}-src.tar.gz"
-  java -jar target/rat.jar -E .rat-excludes -d target/source-unpack
-
-  echo "checking asf licensing requirements for convenience binary '$(pwd)/${bin_file}'."
-  rm -rf target/bin-unpack 2>/dev/null || true
-  mkdir target/bin-unpack
-  tar -C target/bin-unpack -xzf "${bin_file}"
-  java -jar target/rat.jar -E .rat-excludes -d target/bin-unpack
-fi
-echo "All Done!"
-echo "Find your output at:"
-if [ "${release}" = "true" ] && [ -f "target/yetus-${YETUS_VERSION}-src.tar.gz" ]; then
-  echo "    $(pwd)/target/yetus-${YETUS_VERSION}-src.tar.gz"
-fi
-if [ -f "${bin_file}" ]; then
-  echo "    $(pwd)/${bin_file}"
-fi

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/hooks/build
----------------------------------------------------------------------
diff --git a/hooks/build b/hooks/build
new file mode 100755
index 0000000..21876dc
--- /dev/null
+++ b/hooks/build
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ -n "${DOCKERFILE_PATH}" ]]; then
+  docker build -t "${IMAGE_NAME}" \
+	--build-arg DOCKER_TAG="${DOCKER_TAG}" \
+	--build-arg DOCKER_REPO="${DOCKER_REPO}" \
+	-f "${DOCKERFILE_PATH}"
+else
+  docker build -t "${IMAGE_NAME}" \
+	--build-arg DOCKER_TAG="${DOCKER_TAG}" \
+	--build-arg DOCKER_REPO="${DOCKER_REPO}" \
+	 .
+fi

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..334c807
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,336 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                      http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache</groupId>
+    <artifactId>apache</artifactId>
+    <version>21</version>
+    <relativePath/>
+    <!-- no parent resolution -->
+  </parent>
+  <groupId>org.apache.yetus</groupId>
+  <artifactId>yetus-project</artifactId>
+  <version>0.9.0-SNAPSHOT</version>
+  <description>Project-wide definitions for Apache Yetus</description>
+  <name>Apache Yetus Project</name>
+  <packaging>pom</packaging>
+
+  <url>https://yetus.apache.org</url>
+  <inceptionYear>2015</inceptionYear>
+  <mailingLists>
+    <mailingList>
+      <name>Apache Yetus Dev List</name>
+      <subscribe>dev-subscribe@yetus.apache.org</subscribe>
+      <unsubscribe>dev-unsubscribe@yetus.apache.org</unsubscribe>
+      <post>dev@yetus.apache.org</post>
+      <archive>https://mail-archives.apache.org/mod_mbox/yetus-dev/</archive>
+    </mailingList>
+  </mailingLists>
+
+  <properties>
+    <maven.min.version>3.2.0</maven.min.version>
+    <maven.api.version>3.2</maven.api.version>
+    <maven.plugin.api.version>3.5.4</maven.plugin.api.version>
+    <java.min.version>1.8</java.min.version>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <maven.compiler.target>1.8</maven.compiler.target>
+    <extra.enforcer.version>1.0-beta-9</extra.enforcer.version>
+
+    <build-helper-maven-plugin.version>3.0.0</build-helper-maven-plugin.version>
+    <checkstyle.version>8.14</checkstyle.version>
+    <commons.io.version>2.6</commons.io.version>
+    <exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
+    <jdiff.version>1.0.9</jdiff.version>
+    <jython-compile-maven-plugin.version>2.0</jython-compile-maven-plugin.version>
+    <jython-shaded.version>2.7.1</jython-shaded.version>
+    <maven-checkstyle-plugin.version>3.0.0</maven-checkstyle-plugin.version>
+    <maven-project-info-reports-plugin.version>3.0.0</maven-project-info-reports-plugin.version>
+    <spotbugs-maven-plugin.version>3.1.7</spotbugs-maven-plugin.version>
+
+     <sourceReleaseAssemblyDescriptor>source-release-tar</sourceReleaseAssemblyDescriptor>
+
+  </properties>
+
+  <scm>
+    <connection>scm:git:git://git.apache.org/yetus.git</connection>
+    <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/yetus.git</developerConnection>
+    <url>https://git-wip-us.apache.org/repos/asf?p=yetus.git</url>
+  </scm>
+  <issueManagement>
+    <system>JIRA</system>
+    <url>https://issues.apache.org/jira/browse/YETUS</url>
+  </issueManagement>
+
+  <build>
+
+    <pluginManagement>
+      <plugins>
+
+        <plugin>
+          <groupId>org.codehaus.mojo</groupId>
+          <artifactId>exec-maven-plugin</artifactId>
+          <version>${exec-maven-plugin.version}</version>
+        </plugin>
+
+        <plugin>
+          <groupId>com.github.spotbugs</groupId>
+          <artifactId>spotbugs-maven-plugin</artifactId>
+          <version>${spotbugs-maven-plugin.version}</version>
+        </plugin>
+
+
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-checkstyle-plugin</artifactId>
+          <dependencies>
+            <dependency>
+              <groupId>com.puppycrawl.tools</groupId>
+              <artifactId>checkstyle</artifactId>
+              <version>${checkstyle.version}</version>
+            </dependency>
+          </dependencies>
+          <executions>
+            <execution>
+              <id>validate</id>
+              <phase>validate</phase>
+              <configuration>
+                <configLocation>checkstyle.xml</configLocation>
+                <encoding>UTF-8</encoding>
+                <consoleOutput>true</consoleOutput>
+                <failsOnError>true</failsOnError>
+                <linkXRef>false</linkXRef>
+              </configuration>
+              <goals>
+                <goal>check</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin>
+
+        <plugin>
+          <artifactId>maven-clean-plugin</artifactId>
+          <configuration>
+            <failOnError>false</failOnError>
+          </configuration>
+        </plugin>
+
+        <!-- plugin>
+          <groupId>org.apache.rat</groupId>
+          <artifactId>apache-rat-plugin</artifactId>
+          <version>${apache-rat-plugin.version}</version>
+          <executions>
+            <execution>
+              <phase>verify</phase>
+              <goals>
+                <goal>check</goal>
+              </goals>
+            </execution>
+          </executions>
+        </plugin -->
+
+
+      </plugins>
+    </pluginManagement>
+
+
+    <plugins>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-enforcer-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>extra-enforcer-rules</artifactId>
+            <version>${extra.enforcer.version}</version>
+          </dependency>
+        </dependencies>
+        <!-- version set by parent -->
+        <executions>
+          <execution>
+            <id>enforce</id>
+            <goals>
+              <goal>enforce</goal>
+            </goals>
+            <configuration>
+              <rules>
+                <!-- The earliest maven version we verify builds for via ASF Jenkins -->
+                <!-- Might be obviated by move to gradle -->
+                <requireMavenVersion>
+                  <version>[${maven.min.version},)</version>
+                  <message>Maven is out of date.
+  Yetus requires at least version ${maven.min.version} of Maven to properly build from source.
+  You appear to be using an older version. You can use either "mvn -version" or
+  "mvn enforcer:display-info" to verify what version is active.
+  See the contributor guide on building for more information: ${project.url}/contribute/
+                  </message>
+                </requireMavenVersion>
+                <!-- The earliest JVM version we verify builds for via ASF Jenkins -->
+                <requireJavaVersion>
+                  <version>[${java.min.version},)</version>
+                  <message>Java is out of date.
+  Yetus requires at least version ${java.min.version} of the JDK to properly build from source.
+  You appear to be using an older version. You can use either "mvn -version" or
+  "mvn enforcer:display-info" to verify what version is active.
+  See the contributor guide on building for more information: ${project.url}/contribute/
+                  </message>
+                </requireJavaVersion>
+              </rules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludeSubprojects>false</excludeSubprojects>
+          <excludes>
+            <exclude>.pylintrc</exclude>
+            <exclude>.rubocop.yml</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+
+    </plugins>
+  </build>
+
+  <profiles>
+    <profile>
+      <id>gpg2</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-gpg-plugin</artifactId>
+            <configuration>
+              <executable>gpg2</executable>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>jdk1.8</id>
+      <activation>
+        <jdk>1.8</jdk>
+      </activation>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>jdk.tools</groupId>
+            <artifactId>jdk.tools</artifactId>
+            <version>1.8</version>
+            <scope>system</scope>
+            <systemPath>${java.home}/../lib/tools.jar</systemPath>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
+    </profile>
+  </profiles>
+
+  <reporting>
+    <plugins>
+      <plugin>
+        <artifactId>maven-project-info-reports-plugin</artifactId>
+        <reportSets>
+          <reportSet>
+            <configuration>
+              <skip>true</skip>
+            </configuration>
+          </reportSet>
+        </reportSets>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <inherited>false</inherited>
+        <reportSets>
+          <reportSet>
+            <id>aggregate</id>
+            <configuration>
+              <maxmemory>1024m</maxmemory>
+              <quiet>true</quiet>
+              <verbose>false</verbose>
+              <source>${maven.compile.source}</source>
+              <charset>${maven.compile.encoding}</charset>
+              <reportOutputDirectory>${project.build.directory}/site</reportOutputDirectory>
+              <destDir>yetus-project/api</destDir>
+                             <doclet>org.apache.yetus.audience.tools.IncludePublicAnnotationsStandardDoclet</doclet>
+              <docletArtifacts>
+                <docletArtifact>
+                  <groupId>org.apache.yetus</groupId>
+                  <artifactId>audience-annotations</artifactId>
+                  <version>${project.version}</version>
+                </docletArtifact>
+              </docletArtifacts>
+              <useStandardDocletOptions>true</useStandardDocletOptions>
+
+              <!-- switch on dependency-driven aggregation -->
+              <includeDependencySources>false</includeDependencySources>
+
+              <dependencySourceIncludes>
+                <!-- include ONLY dependencies I control -->
+                <dependencySourceInclude>org.apache.hadoop:hadoop-annotations</dependencySourceInclude>
+              </dependencySourceIncludes>
+
+            </configuration>
+            <reports>
+              <report>aggregate</report>
+            </reports>
+          </reportSet>
+        </reportSets>
+      </plugin>
+
+      <!-- plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+        <version>${maven-checkstyle-plugin.version}</version>
+        <reportSets>
+          <reportSet>
+            <reports>
+              <report>checkstyle</report>
+            </reports>
+          </reportSet>
+        </reportSets>
+      </plugin -->
+    </plugins>
+  </reporting>
+
+  <modules>
+    <module>yetus-minimaven-plugin</module>
+    <module>yetus-assemblies</module>
+    <module>audience-annotations-component</module>
+    <module>precommit</module>
+    <module>releasedocmaker</module>
+    <module>shelldocs</module>
+    <module>asf-site-src</module>
+    <module>yetus-dist</module>
+    <module>yetus-maven-plugin</module>
+  </modules>
+
+</project>

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/README.md
----------------------------------------------------------------------
diff --git a/precommit/coprocs.d/README.md b/precommit/coprocs.d/README.md
deleted file mode 100755
index e960cad..0000000
--- a/precommit/coprocs.d/README.md
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#############################
-##
-## bash v4+ coproc routines
-##
-#############################
-
-# bash v3 and lower will treat coproc commands as syntax errors
-# therefore, ALL functions which activate coprocs are located
-# here
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/e_a_r_helper.sh
----------------------------------------------------------------------
diff --git a/precommit/coprocs.d/e_a_r_helper.sh b/precommit/coprocs.d/e_a_r_helper.sh
deleted file mode 100755
index 3df8293..0000000
--- a/precommit/coprocs.d/e_a_r_helper.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# SHELLDOC-IGNORE
-
-## @description helper function for echo_and_redirect
-## @audience private
-## @stability evolving
-## @replaceable no
-function e_a_r_helper
-{
-  declare logfile=$1
-  shift
-  declare params=("${@}")
-
-  echo "Launching yrr_coproc" >> "${COPROC_LOGFILE}"
-  # shellcheck disable=SC2034
-  coproc yrr_coproc {
-    ulimit -Su "${PROC_LIMIT}"
-    yetus_run_and_redirect "${logfile}" "${params[@]}"
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/process_counter.sh
----------------------------------------------------------------------
diff --git a/precommit/coprocs.d/process_counter.sh b/precommit/coprocs.d/process_counter.sh
deleted file mode 100755
index 4f944bb..0000000
--- a/precommit/coprocs.d/process_counter.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# SHELLDOC-IGNORE
-
-function process_counter_coproc_start
-{
-  if [[ "${OSTYPE}" = Linux && "${DOCKERMODE}" = true ]]; then
-    # this is really only even remotely close to
-    # accurate under Docker, for the time being.
-
-    echo "Launching process_counter_coproc" >> "${COPROC_LOGFILE}"
-    # shellcheck disable=SC2034
-    coproc process_counter_coproc {
-      declare threadcount
-      declare maxthreadcount
-      declare cmd
-
-      sleep 2
-      while true; do
-        threadcount=$(ps -L -u "${USER_ID}" -o lwp 2>/dev/null | wc -l)
-        if [[ ${threadcount} -gt ${maxthreadcount} ]]; then
-          maxthreadcount="${threadcount}"
-          echo "${maxthreadcount}" > "${PATCH_DIR}/threadcounter.txt"
-        fi
-        read -r -t 2 cmd
-        case "${cmd}" in
-          exit)
-            exit 0
-          ;;
-        esac
-      done
-    }
-  fi
-}

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/coprocs.d/reaper.sh
----------------------------------------------------------------------
diff --git a/precommit/coprocs.d/reaper.sh b/precommit/coprocs.d/reaper.sh
deleted file mode 100755
index 4674130..0000000
--- a/precommit/coprocs.d/reaper.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# SHELLDOC-IGNORE
-
-function reaper_coproc_start
-{
-  if [[ "${REAPER_MODE}" != "off" ]]; then
-
-    echo "Launching reaper_coproc" >> "${COPROC_LOGFILE}"
-
-    # shellcheck disable=SC2034
-    coproc reaper_coproc {
-      reaper_coproc_func
-    }
-  fi
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/00-yetuslib.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/00-yetuslib.sh b/precommit/core.d/00-yetuslib.sh
deleted file mode 100755
index 983dfe6..0000000
--- a/precommit/core.d/00-yetuslib.sh
+++ /dev/null
@@ -1,331 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# we need to declare this globally as an array, which can only
-# be done outside of a function
-declare -a YETUS_OPTION_USAGE
-
-## @description  Print a message to stderr
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        string
-function yetus_error
-{
-  echo "$*" 1>&2
-}
-
-## @description  Print a message to stderr if --debug is turned on
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        string
-function yetus_debug
-{
-  if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then
-    echo "[$(date) DEBUG]: $*" 1>&2
-  fi
-}
-
-## @description  Given variable $1 delete $2 from it
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function yetus_delete_entry
-{
-  if [[ ${!1} =~ \ ${2}\  ]] ; then
-    yetus_debug "Removing ${2} from ${1}"
-    eval "${1}"=\""${!1// ${2} }"\"
-  fi
-}
-
-## @description  Given variable $1 add $2 to it
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function yetus_add_entry
-{
-  if [[ ! ${!1} =~ \ ${2}\  ]] ; then
-    yetus_debug "Adding ${2} to ${1}"
-    #shellcheck disable=SC2140
-    eval "${1}"=\""${!1} ${2} "\"
-  fi
-}
-
-## @description  Given variable $1 determine if $2 is in it
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @return       0 = yes, 1 = no
-function yetus_verify_entry
-{
-  [[ ${!1} =~ \ ${2}\  ]]
-}
-
-## @description  run the command, sending stdout and stderr to the given filename
-## @audience     public
-## @stability    stable
-## @param        filename
-## @param        command
-## @param        [..]
-## @replaceable  no
-## @return       $?
-function yetus_run_and_redirect
-{
-  declare logfile=$1
-  shift
-
-  # to the log
-  {
-    date
-    echo "cd $(pwd)"
-    echo "${*}"
-  } >> "${logfile}"
-  # run the actual command
-  "${@}" >> "${logfile}" 2>&1
-}
-
-## @description  Given a filename or dir, return the absolute version of it
-## @audience     public
-## @stability    stable
-## @param        fsobj
-## @replaceable  no
-## @return       0 success
-## @return       1 failure
-## @return       stdout abspath
-function yetus_abs
-{
-  declare obj=$1
-  declare dir
-  declare fn
-
-  if [[ ! -e ${obj} ]]; then
-    return 1
-  elif [[ -d ${obj} ]]; then
-    dir=${obj}
-  else
-    dir=$(dirname -- "${obj}")
-    fn=$(basename -- "${obj}")
-    fn="/${fn}"
-  fi
-
-  dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P)
-  if [[ $? = 0 ]]; then
-    echo "${dir}${fn}"
-    return 0
-  fi
-  return 1
-}
-
-## @description  Add a header to the usage output
-## @audience     public
-## @stability    evolving
-## @replaceable  no
-## @param        header
-function yetus_add_header
-{
-  declare text=$1
-
-  #shellcheck disable=SC2034
-  YETUS_USAGE_HEADER="${text}"
-}
-
-## @description  Add an option to the usage output
-## @audience     public
-## @stability    evolving
-## @replaceable  no
-## @param        subcommand
-## @param        subcommanddesc
-function yetus_add_option
-{
-  declare option=$1
-  declare text=$2
-
-  YETUS_OPTION_USAGE[${YETUS_OPTION_USAGE_COUNTER}]="${option}@${text}"
-  ((YETUS_OPTION_USAGE_COUNTER=YETUS_OPTION_USAGE_COUNTER+1))
-}
-
-## @description  Reset the usage information to blank
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function yetus_reset_usage
-{
-  # shellcheck disable=SC2034
-  YETUS_OPTION_USAGE=()
-  YETUS_OPTION_USAGE_COUNTER=0
-}
-
-## @description  Print a screen-size aware two-column output
-## @audience     public
-## @stability    evolving
-## @replaceable  no
-## @param        array
-function yetus_generic_columnprinter
-{
-  declare -a input=("$@")
-  declare -i i=0
-  declare -i counter=0
-  declare line
-  declare text
-  declare option
-  declare giventext
-  declare -i maxoptsize
-  declare -i foldsize
-  declare -a tmpa
-  declare numcols
-
-  if [[ -n "${COLUMNS}" ]]; then
-    numcols=${COLUMNS}
-  else
-    numcols=$(tput cols) 2>/dev/null
-  fi
-
-  if [[ -z "${numcols}"
-     || ! "${numcols}" =~ ^[0-9]+$ ]]; then
-    numcols=75
-  else
-    ((numcols=numcols-5))
-  fi
-
-  while read -r line; do
-    tmpa[${counter}]=${line}
-    ((counter=counter+1))
-    option=$(echo "${line}" | cut -f1 -d'@')
-    if [[ ${#option} -gt ${maxoptsize} ]]; then
-      maxoptsize=${#option}
-    fi
-  done < <(for text in "${input[@]}"; do
-    echo "${text}"
-  done | sort)
-
-  i=0
-  ((foldsize=numcols-maxoptsize))
-
-  until [[ $i -eq ${#tmpa[@]} ]]; do
-    option=$(echo "${tmpa[$i]}" | cut -f1 -d'@')
-    giventext=$(echo "${tmpa[$i]}" | cut -f2 -d'@')
-
-    while read -r line; do
-      printf "%-${maxoptsize}s   %-s\n" "${option}" "${line}"
-      option=" "
-    done < <(echo "${giventext}"| fold -s -w ${foldsize})
-    ((i=i+1))
-  done
-}
-
-## @description  Convert a comma-delimited string to an array
-## @audience     public
-## @stability    evolving
-## @replaceable  no
-## @param        arrayname
-## @param        string
-function yetus_comma_to_array
-{
-  declare var=$1
-  declare string=$2
-
-  oldifs="${IFS}"
-  IFS=',' read -r -a "${var}" <<< "${string}"
-  IFS="${oldifs}"
-}
-
-## @description  Check if an array has a given value
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        element
-## @param        array
-## @returns      0 = yes
-## @returns      1 = no
-function yetus_array_contains
-{
-  declare element=$1
-  shift
-  declare val
-
-  if [[ "$#" -eq 0 ]]; then
-    return 1
-  fi
-
-  for val in "${@}"; do
-    if [[ "${val}" == "${element}" ]]; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-## @description  Add the element if it is not
-## @description  present in the given array
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        arrayname
-## @param        element
-function yetus_add_array_element
-{
-  declare arrname=$1
-  declare add=$2
-
-  declare arrref="${arrname}[@]"
-  declare array=("${!arrref}")
-
-  if ! yetus_array_contains "${add}" "${array[@]}"; then
-    # shellcheck disable=SC1083,SC2086
-    eval "${arrname}"=\(\"\${array[@]}\" \"${add}\" \)
-    yetus_debug "$1 accepted $2"
-  else
-    yetus_debug "$1 declined $2"
-  fi
-}
-
-## @description  Sort an array by its elements
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        arrayvar
-function yetus_sort_array
-{
-  declare arrname=$1
-  declare arrref="${arrname}[@]"
-  declare array=("${!arrref}")
-
-  declare globstatus
-  declare oifs
-  declare -a sa
-
-  globstatus=$(set -o | grep noglob | awk '{print $NF}')
-
-  if [[ -n ${IFS} ]]; then
-    oifs=${IFS}
-  fi
-  set -f
-  # shellcheck disable=SC2034
-  IFS=$'\n' sa=($(sort <<<"${array[*]}"))
-  # shellcheck disable=SC1083
-  eval "${arrname}"=\(\"\${sa[@]}\"\)
-
-  if [[ -n "${oifs}" ]]; then
-    IFS=${oifs}
-  else
-    unset IFS
-  fi
-
-  if [[ "${globstatus}" = off ]]; then
-    set +f
-  fi
-}

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/01-common.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/01-common.sh b/precommit/core.d/01-common.sh
deleted file mode 100755
index e8dee09..0000000
--- a/precommit/core.d/01-common.sh
+++ /dev/null
@@ -1,615 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description  Setup the default global variables
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function common_defaults
-{
-  #shellcheck disable=SC2034
-  BASEDIR=$(pwd)
-  BUGSYSTEMS=""
-  BUILDTOOL="nobuild"
-  BUILDTOOLS=""
-  #shellcheck disable=SC2034
-  EXEC_MODES=""
-  #shellcheck disable=SC2034
-  JENKINS=false
-  LOAD_SYSTEM_PLUGINS=true
-  #shellcheck disable=SC2034
-  OFFLINE=false
-  OSTYPE=$(uname -s)
-  #shellcheck disable=SC2034
-  PATCH_BRANCH=""
-  PATCH_BRANCH_DEFAULT="master"
-  #shellcheck disable=SC2034
-  PATCH_DRYRUNMODE=false
-  PATCH_DIR=/tmp
-  while [[ -e ${PATCH_DIR} ]]; do
-    PATCH_DIR=/tmp/yetus-${RANDOM}.${RANDOM}
-  done
-  #shellcheck disable=SC2034
-  PATCH_METHOD=""
-  #shellcheck disable=SC2034
-  PATCH_METHODS=("gitapply" "patchcmd")
-  #shellcheck disable=SC2034
-  PATCH_LEVEL=0
-  #shellcheck disable=SC2034
-  PATCH_SYSTEM=""
-  PROJECT_NAME=unknown
-  RESULT=0
-  #shellcheck disable=SC2034
-  ROBOT=false
-  #shellcheck disable=SC2034
-  SENTINEL=false
-  #shellcheck disable=SC2034
-  TESTTYPES=""
-  TESTFORMATS=""
-  USER_PLUGIN_DIR=""
-
-  #shellcheck disable=SC2034
-  YETUS_SHELL_SCRIPT_DEBUG=false
-
-  # Solaris needs POSIX and GNU, not SVID
-  case ${OSTYPE} in
-    SunOS)
-      AWK=${AWK:-/usr/xpg4/bin/awk}
-      CURL=${CURL:-curl}
-      DIFF=${DIFF:-/usr/gnu/bin/diff}
-      FILE=${FILE:-file}
-      GIT=${GIT:-git}
-      GREP=${GREP:-/usr/xpg4/bin/grep}
-      PATCH=${PATCH:-/usr/gnu/bin/patch}
-      SED=${SED:-/usr/xpg4/bin/sed}
-    ;;
-    *)
-      AWK=${AWK:-awk}
-      CURL=${CURL:-curl}
-      DIFF=${DIFF:-diff}
-      FILE=${FILE:-file}
-      GIT=${GIT:-git}
-      GREP=${GREP:-grep}
-      PATCH=${PATCH:-patch}
-      SED=${SED:-sed}
-    ;;
-  esac
-
-  RSYNC=${RSYNC:-rsync}
-}
-
-## @description  Interpret the common command line parameters used by test-patch,
-## @description  smart-apply-patch, and the bug system plugins
-## @audience     private
-## @stability    stable
-## @replaceable  no
-## @param        $@
-## @return       May exit on failure
-function common_args
-{
-  declare i
-  declare showhelp=false
-  declare showversion=false
-  declare version
-
-  for i in "$@"; do
-    case ${i} in
-      --awk-cmd=*)
-        AWK=${i#*=}
-      ;;
-      --basedir=*)
-        #shellcheck disable=SC2034
-        BASEDIR=${i#*=}
-      ;;
-      --branch=*)
-        #shellcheck disable=SC2034
-        PATCH_BRANCH=${i#*=}
-      ;;
-      --branch-default=*)
-        #shellcheck disable=SC2034
-        PATCH_BRANCH_DEFAULT=${i#*=}
-      ;;
-      --curl-cmd=*)
-        CURL=${i#*=}
-      ;;
-      --debug)
-        #shellcheck disable=SC2034
-        YETUS_SHELL_SCRIPT_DEBUG=true
-      ;;
-      --diff-cmd=*)
-        DIFF=${i#*=}
-      ;;
-      --file-cmd=*)
-        FILE=${i#*=}
-      ;;
-      --git-cmd=*)
-        GIT=${i#*=}
-      ;;
-      --grep-cmd=*)
-        GREP=${i#*=}
-      ;;
-      --help|-help|-h|help|--h|--\?|-\?|\?)
-        showhelp=true
-      ;;
-      --list-plugins)
-        list_plugins
-        exit 0
-      ;;
-      --offline)
-        #shellcheck disable=SC2034
-        OFFLINE=true
-      ;;
-      --patch-cmd=*)
-        PATCH=${i#*=}
-      ;;
-      --patch-dir=*)
-        PATCH_DIR=${i#*=}
-      ;;
-      --plugins=*)
-        ENABLED_PLUGINS=${i#*=}
-        ENABLED_PLUGINS=${ENABLED_PLUGINS//,/ }
-      ;;
-      --project=*)
-        PROJECT_NAME=${i#*=}
-      ;;
-      --rsync-cmd=*)
-        RSYNC=${i#*=}
-      ;;
-      --skip-system-plugins)
-        LOAD_SYSTEM_PLUGINS=false
-      ;;
-      --sed-cmd=*)
-        SED=${i#*=}
-      ;;
-      --user-plugins=*)
-        USER_PLUGIN_DIR=${i#*=}
-      ;;
-      --version)
-        showversion=true
-      ;;
-      *)
-      ;;
-    esac
-  done
-  if [[ ${showhelp} == true ]]; then
-    yetus_usage
-    exit 0
-  fi
-  if [[ ${showversion} == true ]]; then
-    cat "${BINDIR}/../VERSION"
-    exit 0
-  fi
-
-  # Absolutely require v1.7.3 or higher
-  # versions lower than this either have bugs with
-  # git apply or don't support all the
-  # expected options
-  version=$(${GIT} --version)
-
-  if [[ $? != 0 ]]; then
-    yetus_error "ERROR: ${GIT} failed during version detection."
-    exit 1
-  fi
-
-  # shellcheck disable=SC2016
-  version=$(echo "${version}" | ${AWK} '{print $NF}')
-  if [[ ${version} =~ ^0
-     || ${version} =~ ^1.[0-6]
-     || ${version} =~ ^1.7.[0-2]$
-    ]]; then
-    yetus_error "ERROR: ${GIT} v1.7.3 or higher is required (found ${version})."
-    exit 1
-  fi
-}
-
-## @description  List all installed plug-ins, regardless of whether
-## @description  they have been enabled
-## @audience     public
-## @stability    evolving
-## @replaceable  no
-function list_plugins
-{
-  declare plugintype
-  declare name
-
-  ENABLED_PLUGINS="all"
-  importplugins
-
-  printf "Reminder: every plug-in may be enabled via 'all'.\n\n"
-  for plugintype in BUILDTOOLS TESTTYPES BUGSYSTEMS TESTFORMATS; do
-    printf "%s:\n\t" ${plugintype}
-    for name in ${!plugintype}; do
-      printf "%s " ${name}
-    done
-    echo ""
-  done
-}
-
-## @description  Let plugins also get a copy of the arguments
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function parse_args_plugins
-{
-  declare plugin
-
-  for plugin in ${TESTTYPES} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOLS}; do
-    if declare -f ${plugin}_parse_args >/dev/null 2>&1; then
-      yetus_debug "Running ${plugin}_parse_args"
-      #shellcheck disable=SC2086
-      ${plugin}_parse_args "$@"
-      (( RESULT = RESULT + $? ))
-    fi
-  done
-}
-
-## @description  Initialize all enabled plugins
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function plugins_initialize
-{
-  declare plugin
-
-  for plugin in ${TESTTYPES} ${BUGSYSTEMS} ${TESTFORMATS} ${BUILDTOOL}; do
-    if declare -f ${plugin}_initialize >/dev/null 2>&1; then
-      yetus_debug "Running ${plugin}_initialize"
-      #shellcheck disable=SC2086
-      ${plugin}_initialize
-      (( RESULT = RESULT + $? ))
-    fi
-  done
-}
-
-## @description  Determine if a plugin was enabled by the user
-## @description  ENABLED_PLUGINS must be defined
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        test
-function verify_plugin_enabled
-{
-  declare toadd=$1
-  declare bar
-  declare idx
-  declare strip
-  declare stridx
-
-  yetus_debug "Testing if $1 has been enabled by user"
-
-  bar=""
-  for idx in ${ENABLED_PLUGINS}; do
-    stridx=${idx// }
-    yetus_debug "verify_plugin_enabled: processing ${stridx}"
-    case ${stridx} in
-      all)
-        bar=${toadd}
-      ;;
-      -*)
-        strip=${stridx#-}
-        if [[ ${strip} = "${toadd}" ]]; then
-          bar=""
-        fi
-        ;;
-      +*|*)
-        strip=${stridx#+}
-        if [[ ${strip} = "${toadd}" ]]; then
-          bar=${toadd}
-        fi
-      ;;
-    esac
-  done
-
-  if [[ -n ${bar} ]]; then
-    yetus_debug "Post-parsing: checking ${bar} = ${toadd}"
-  fi
-  [[ ${bar} = "${toadd}" ]]
-}
-
-## @description  Personality-defined plug-in list
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        plug-in list string
-function personality_plugins
-{
-  if [[ -z "${ENABLED_PLUGINS}" ]]; then
-    ENABLED_PLUGINS="$1"
-    ENABLED_PLUGINS=${ENABLED_PLUGINS//,/ }
-    yetus_debug "Using personality plug-in list: ${ENABLED_PLUGINS}"
-  fi
-}
-
-## @description  Add the given test type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        test
-function add_test
-{
-  if verify_plugin_enabled "${1}"; then
-    yetus_add_entry NEEDED_TESTS "${1}"
-  fi
-}
-
-## @description  Remove the given test type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        test
-function delete_test
-{
-  yetus_delete_entry NEEDED_TESTS "${1}"
-}
-
-## @description  Verify if a given test was requested
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        test
-## @return       0 = yes
-## @return       1 = no
-function verify_needed_test
-{
-  yetus_verify_entry NEEDED_TESTS "${1}"
-}
-
-## @description  Add the given test type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        plugin
-function add_test_type
-{
-  if verify_plugin_enabled "${1}"; then
-    yetus_add_entry TESTTYPES "${1}"
-  fi
-}
-
-## @description  Remove the given test type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        plugin
-function delete_test_type
-{
-  yetus_delete_entry TESTTYPES "${1}"
-}
-
-## @description  Add the given bugsystem type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        bugsystem
-function add_bugsystem
-{
-  if verify_plugin_enabled "${1}"; then
-    yetus_add_entry BUGSYSTEMS "${1}"
-  fi
-}
-
-## @description  Remove the given bugsystem type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        bugsystem
-function delete_bugsystem
-{
-  yetus_delete_entry BUGSYSTEMS "${1}"
-}
-
-## @description  Add the given test format type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        test format
-function add_test_format
-{
-  if verify_plugin_enabled "${1}"; then
-    yetus_add_entry TESTFORMATS "${1}"
-  fi
-}
-
-## @description  Remove the given test format type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        test format
-function delete_test_format
-{
-  yetus_delete_entry TESTFORMATS "${1}"
-}
-
-## @description  Add the given build tool type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        build tool
-function add_build_tool
-{
-  if verify_plugin_enabled "${1}"; then
-    yetus_add_entry BUILDTOOLS "${1}"
-  fi
-}
-
-## @description  Remove the given build tool type
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        build tool
-function delete_build_tool
-{
-  yetus_delete_entry BUILDTOOLS "${1}"
-}
-
-## @description  Import content from test-patch.d and optionally
-## @description  from user provided plugin directory
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function importplugins
-{
-  local i
-  local plugin
-  local files=()
-
-  if [[ ${LOAD_SYSTEM_PLUGINS} == "true" ]]; then
-    if [[ -d "${BINDIR}/test-patch.d" ]]; then
-      files=(${BINDIR}/test-patch.d/*.sh)
-    fi
-  fi
-
-  if [[ -n "${USER_PLUGIN_DIR}" && -d "${USER_PLUGIN_DIR}" ]]; then
-    yetus_debug "Loading user provided plugins from ${USER_PLUGIN_DIR}"
-    files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
-  fi
-
-  if [[ -n ${PERSONALITY} && ! -f ${PERSONALITY} ]]; then
-    yetus_error "ERROR: Can't find ${PERSONALITY} to import."
-    unset PERSONALITY
-  fi
-
-  if [[ -z ${PERSONALITY}
-      && -f "${BINDIR}/personality/${PROJECT_NAME}.sh"
-      && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then
-    yetus_debug "Using project personality."
-    PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
-  fi
-
-  if [[ -n ${PERSONALITY} && -f ${PERSONALITY} ]]; then
-    yetus_debug "Importing ${PERSONALITY}"
-    # shellcheck disable=SC1090
-    . "${PERSONALITY}"
-  fi
-
-  for i in "${files[@]}"; do
-    if [[ -f ${i} ]]; then
-      yetus_debug "Importing ${i}"
-      #shellcheck disable=SC1090
-      . "${i}"
-    fi
-  done
-
-  if declare -f personality_globals > /dev/null; then
-    personality_globals
-  fi
-}
-
-## @description  Print the plugin's usage info
-## @audience     public
-## @stability    evolving
-## @replaceable  no
-## @param        array
-function plugin_usage_output
-{
-  echo ""
-  echo "${YETUS_USAGE_HEADER}"
-  echo ""
-}
-
-## @description  Verifies the existence of a command
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        commandname
-## @param        commandpath
-## @return       0 = ok
-## @return       1 = error
-function verify_command
-{
-  local cmd_name="$1"
-  local cmd_path="$2"
-
-  if [[ -z ${cmd_path} ]]; then
-    yetus_error "executable for '${cmd_name}' was not specified."
-    return 1
-  fi
-  if [[ ! "${cmd_path}" =~ / ]]; then
-    cmd_path=$(command -v "${cmd_path}")
-  fi
-  if [[ ! -f ${cmd_path} ]]; then
-    yetus_error "executable '${cmd_path}' for '${cmd_name}' does not exist."
-    return 1
-  fi
-  if [[ ! -x ${cmd_path} ]]; then
-    yetus_error "executable '${cmd_path}' for '${cmd_name}' is not executable."
-    return 1
-  fi
-  return 0
-}
-
-## @description  Faster dirname, given the assumption that
-## @description  dirs are always absolute (e.g., start with /)
-## @description  DO NOT USE with relative paths or where
-## @description  assumption may not be valid!
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        fileobj
-function faster_dirname
-{
-  declare o=$1
-
-  if [[ "${o}" =~ / ]]; then
-    echo "${o%/*}"
-  else
-    echo .
-  fi
-}
-
-## @description  Set the USER_NAME, USER_ID, and GROUP_ID env vars
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function determine_user
-{
-  # On the Apache Jenkins hosts, $USER is pretty much untrustable because
-  # something sets it to an account that doesn't actually exist.
-  # Instead, we need to try and override it with something that's
-  # probably close to reality.
-  if [[ ${TESTPATCHMODE} =~ jenkins ]]; then
-    USER=$(id | cut -f2 -d\( | cut -f1 -d\))
-  fi
-
-  USER_NAME=${SUDO_USER:=$USER}
-  # shellcheck disable=SC2034
-  USER_ID=$(id -u "${USER_NAME}")
-  # shellcheck disable=SC2034
-  GROUP_ID=$(id -g "${USER_NAME}")
-}
-
-## @description  Kill a process id
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        pid
-function pid_kill
-{
-  declare pid=$1
-  declare cmd
-  declare kill_timeout=3
-
-  kill "${pid}" >/dev/null 2>&1
-  sleep "${kill_timeout}"
-  if kill -0 "${pid}" > /dev/null 2>&1; then
-    yetus_error "WARNING: ${pid} did not stop gracefully after ${kill_timeout} second(s): Trying to kill with kill -9"
-    kill -9 "${pid}" >/dev/null 2>&1
-  fi
-  if ps -p "${pid}" > /dev/null 2>&1; then
-    cmd=$(ps -o args "${pid}")
-    yetus_error "ERROR: Unable to kill ${pid}: ${cmd}"
-  fi
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/builtin-bugsystem.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/builtin-bugsystem.sh b/precommit/core.d/builtin-bugsystem.sh
deleted file mode 100755
index 0aeddf1..0000000
--- a/precommit/core.d/builtin-bugsystem.sh
+++ /dev/null
@@ -1,185 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# This bug system handles the output on the screen.
-
-add_bugsystem console
-
-CONSOLE_USE_BUILD_URL=false
-
-## @description  Print out the finished details on the console
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        runresult
-## @return       0 on success
-## @return       1 on failure
-function console_finalreport
-{
-  declare result=$1
-  shift
-  declare i=0
-  declare ourstring
-  declare vote
-  declare subs
-  declare ela
-  declare comment
-  declare commentfile1="${PATCH_DIR}/comment.1"
-  declare commentfile2="${PATCH_DIR}/comment.2"
-  declare normaltop
-  declare line
-  declare seccoladj=0
-  declare spcfx=${PATCH_DIR}/spcl.txt
-  declare calctime
-
-  if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then
-    exec 6>&1
-    exec >"${CONSOLE_REPORT_FILE}"
-  fi
-
-  if [[ ${result} == 0 ]]; then
-    if [[ ${ROBOT} == false ]]; then
-      if declare -f ${PROJECT_NAME}_console_success >/dev/null; then
-        "${PROJECT_NAME}_console_success" > "${spcfx}"
-      else
-        {
-          printf "IF9fX18gICAgICAgICAgICAgICAgICAgICAgICAgICAgICBfIAovIF9fX3wg";
-          printf "XyAgIF8gIF9fXyBfX18gX19fICBfX18gX19ffCB8ClxfX18gXHwgfCB8IHwv";
-          printf "IF9fLyBfXy8gXyBcLyBfXy8gX198IHwKIF9fXykgfCB8X3wgfCAoX3wgKF98";
-          printf "ICBfXy9cX18gXF9fIFxffAp8X19fXy8gXF9fLF98XF9fX1xfX19cX19ffHxf";
-          printf "X18vX19fKF8pCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg";
-          printf "ICAK";
-        } > "${spcfx}"
-      fi
-    fi
-    printf "\n\n+1 overall\n\n"
-  else
-    if [[ ${ROBOT} == false ]]; then
-      if declare -f ${PROJECT_NAME}_console_failure >/dev/null; then
-        "${PROJECT_NAME}_console_failure" > "${spcfx}"
-      else
-        {
-          printf "IF9fX19fICAgICBfIF8gICAgICAgICAgICAgICAgXyAKfCAgX19ffF8gXyhf";
-          printf "KSB8XyAgIF8gXyBfXyBfX198IHwKfCB8XyAvIF9gIHwgfCB8IHwgfCB8ICdf";
-          printf "Xy8gXyBcIHwKfCAgX3wgKF98IHwgfCB8IHxffCB8IHwgfCAgX18vX3wKfF98";
-          printf "ICBcX18sX3xffF98XF9fLF98X3wgIFxfX18oXykKICAgICAgICAgICAgICAg";
-          printf "ICAgICAgICAgICAgICAgICAK"
-        } > "${spcfx}"
-      fi
-    fi
-    printf "\n\n-1 overall\n\n"
-  fi
-
-  if [[ -f ${spcfx} ]]; then
-    if which base64 >/dev/null 2>&1; then
-      base64 --decode "${spcfx}" 2>/dev/null
-    elif which openssl >/dev/null 2>&1; then
-      openssl enc -A -d -base64 -in "${spcfx}" 2>/dev/null
-    fi
-    echo
-    echo
-    rm "${spcfx}"
-  fi
-
-  seccoladj=$(findlargest 2 "${TP_VOTE_TABLE[@]}")
-  if [[ ${seccoladj} -lt 10 ]]; then
-    seccoladj=10
-  fi
-
-  seccoladj=$((seccoladj + 2 ))
-  i=0
-  until [[ $i -eq ${#TP_HEADER[@]} ]]; do
-    printf "%s\n" "${TP_HEADER[${i}]}"
-    ((i=i+1))
-  done
-
-  printf "| %s | %*s |  %s   | %s\n" "Vote" ${seccoladj} Subsystem Runtime "Comment"
-  echo "============================================================================"
-  i=0
-  until [[ $i -eq ${#TP_VOTE_TABLE[@]} ]]; do
-    ourstring=$(echo "${TP_VOTE_TABLE[${i}]}" | tr -s ' ')
-    vote=$(echo "${ourstring}" | cut -f2 -d\|)
-    subs=$(echo "${ourstring}"  | cut -f3 -d\|)
-    ela=$(echo "${ourstring}" | cut -f4 -d\|)
-    calctime=$(clock_display "${ela}")
-    comment=$(echo "${ourstring}"  | cut -f5 -d\|)
-
-    echo "${comment}" | fold -s -w $((78-seccoladj-22)) > "${commentfile1}"
-    normaltop=$(head -1 "${commentfile1}")
-    ${SED} -e '1d' "${commentfile1}"  > "${commentfile2}"
-
-    if [[ "${vote}" = "H" ]]; then
-      printf "|      | %*s |            |%-s\n" ${seccoladj} " " "${normaltop}"
-    else
-      printf "| %4s | %*s | %-10s |%-s\n" "${vote}" ${seccoladj} \
-        "${subs}" "${calctime}" "${normaltop}"
-    fi
-    while read -r line; do
-      printf "|      | %*s |            | %-s\n" ${seccoladj} " " "${line}"
-    done < "${commentfile2}"
-
-    ((i=i+1))
-    rm "${commentfile2}" "${commentfile1}" 2>/dev/null
-  done
-
-  if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then
-    seccoladj=$(findlargest 1 "${TP_TEST_TABLE[@]}")
-    printf "\n\n%*s | Tests\n" "${seccoladj}" "Reason"
-    i=0
-    until [[ $i -eq ${#TP_TEST_TABLE[@]} ]]; do
-      ourstring=$(echo "${TP_TEST_TABLE[${i}]}" | tr -s ' ')
-      vote=$(echo "${ourstring}" | cut -f2 -d\|)
-      subs=$(echo "${ourstring}"  | cut -f3 -d\|)
-      printf "%*s | %s\n" "${seccoladj}" "${vote}" "${subs}"
-      ((i=i+1))
-    done
-  fi
-
-  printf "\n\n|| Subsystem || Report/Notes ||\n"
-  echo "============================================================================"
-  i=0
-
-  until [[ $i -eq ${#TP_FOOTER_TABLE[@]} ]]; do
-    if [[ "${CONSOLE_USE_BUILD_URL}" = true &&
-          -n "${BUILD_URL}" ]]; then
-      comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
-                ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
-    else
-      comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
-                ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
-    fi
-    printf "%s\n" "${comment}"
-    ((i=i+1))
-  done
-
-  if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then
-    exec 1>&6 6>&-
-    cat "${CONSOLE_REPORT_FILE}"
-  fi
-}
-
-
-## @description  Give access to the brief report file in docker mode
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function console_docker_support
-{
-  if [[ -n "${CONSOLE_REPORT_FILE}" ]]; then
-    DOCKER_EXTRAARGS+=("-v" "${CONSOLE_REPORT_FILE}:/testptch/console.txt")
-  fi
-}

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/builtin-personality.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/builtin-personality.sh b/precommit/core.d/builtin-personality.sh
deleted file mode 100755
index c473d66..0000000
--- a/precommit/core.d/builtin-personality.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-## @description  Generate a list of all personality modules for a given
-## @description  buildtool for the system to invoke
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function personality_modules
-{
-  if declare -f "${BUILDTOOL}_builtin_personality_modules" >/dev/null; then
-    "${BUILDTOOL}_builtin_personality_modules" "$@"
-  fi
-}
-
-## @description  Generate a list of all personality file tests for a given
-## @description  buildtool for the system to invoke
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function personality_file_tests
-{
-  if declare -f "${BUILDTOOL}_builtin_personality_file_tests" >/dev/null; then
-    "${BUILDTOOL}_builtin_personality_file_tests" "$@"
-  fi
-}

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/precommit/core.d/docker.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh
deleted file mode 100755
index 83efc3b..0000000
--- a/precommit/core.d/docker.sh
+++ /dev/null
@@ -1,661 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DOCKERMODE=false
-DOCKERCMD=$(command -v docker)
-DOCKER_ID=${RANDOM}
-DOCKER_DESTRUCTIVE=true
-DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile"
-DOCKERFAIL="fallback,continue,fail"
-DOCKERSUPPORT=false
-DOCKER_ENABLE_PRIVILEGED=true
-DOCKER_CLEANUP_CMD=false
-DOCKER_MEMORY="4g"
-
-declare -a DOCKER_EXTRAARGS
-
-####
-#### IMPORTANT
-####
-#### If these times are updated, the documentation needs to
-#### be changed too!
-
-# created, stopped, exited, running, for 24 hours
-DOCKER_CONTAINER_PURGE=("86400" "86400" "86400" "86400" )
-
-# keep images for 1 week
-DOCKER_IMAGE_PURGE=604800
-
-## @description  Docker-specific usage
-## @stability    stable
-## @audience     private
-## @replaceable  no
-function docker_usage
-{
-  if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
-    yetus_add_option "--docker" "Spawn a docker container"
-  fi
-  yetus_add_option "--dockercmd=<file>" "Command to use as docker executable (default: '${DOCKERCMD}')"
-  if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
-    yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default: '${DOCKERFILE_DEFAULT}')"
-    yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method order (default: ${DOCKERFAIL})"
-    yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default: '${DOCKER_ENABLE_PRIVILEGED}')"
-  fi
-  yetus_add_option "--dockerdelrep" "In Docker mode, only report image/container deletions, not act on them"
-  if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
-    yetus_add_option "--dockermemlimit=<num>" "Limit a Docker container's memory usage (default: ${DOCKER_MEMORY})"
-  fi
-}
-
-## @description  Docker-specific argument parsing
-## @stability    stable
-## @audience     private
-## @replaceable  no
-## @params       arguments
-function docker_parse_args
-{
-  declare i
-
-  for i in "$@"; do
-    case ${i} in
-      --docker)
-        DOCKERSUPPORT=true
-      ;;
-      --dockercmd=*)
-        #shellcheck disable=SC2034
-        DOCKERCMD=${i#*=}
-      ;;
-      --dockerdelrep)
-        DOCKER_DESTRUCTIVE=false
-      ;;
-      --dockerfile=*)
-        DOCKERFILE=${i#*=}
-      ;;
-      --dockermemlimit=*)
-        DOCKER_MEMORY=${i#*=}
-      ;;
-      --dockermode)
-        DOCKERMODE=true
-      ;;
-      --dockeronfail=*)
-        DOCKERFAIL=${i#*=}
-      ;;
-      --dockerprivd=*)
-        DOCKER_ENABLE_PRIVILEGED=${i#*=}
-      ;;
-    esac
-  done
-}
-
-## @description  Docker initialization pre- and post- re-exec
-## @stability    stable
-## @audience     private
-## @replaceable  no
-function docker_initialize
-{
-  declare dockvers
-
-  # --docker and --dockermode are mutually
-  # exclusive.  --docker is used by the user to
-  # re-exec test-patch in Docker mode.
-  # --dockermode is used by launch-test-patch (which is
-  # run as the Docker EXEC in the Dockerfile,
-  # see elsewhere for more info) to tell test-patch that
-  # it has been restarted already. launch-test-patch
-  # also strips --docker from the command line so that we
-  # don't end up in a loop if the docker image
-  # also has the docker command in it
-
-  # we are already in docker mode
-  if [[ "${DOCKERMODE}" == true ]]; then
-    # DOCKER_VERSION is set by our creator.
-    add_footer_table "Docker" "${DOCKER_VERSION}"
-    return
-  fi
-
-  # docker mode hasn't been requested
-  if [[ "${DOCKERSUPPORT}" != true ]]; then
-    return
-  fi
-
-  # turn DOCKERFAIL into a string composed of numbers
-  # to ease interpretation:  123, 213, 321, ... whatever
-  # some of these combos are non-sensical but that's ok.
-  # we'll treat non-sense as effectively errors.
-  DOCKERFAIL=${DOCKERFAIL//,/ }
-  DOCKERFAIL=${DOCKERFAIL//fallback/1}
-  DOCKERFAIL=${DOCKERFAIL//continue/2}
-  DOCKERFAIL=${DOCKERFAIL//fail/3}
-  DOCKERFAIL=${DOCKERFAIL//[[:blank:]]/}
-
-  if ! docker_exeverify; then
-    if [[ "${DOCKERFAIL}" =~ ^12
-       || "${DOCKERFAIL}" =~ ^2 ]]; then
-      add_vote_table 0 docker "Docker command '${DOCKERCMD}' not found/broken. Disabling docker."
-      DOCKERSUPPORT=false
-    else
-      add_vote_table -1 docker "Docker command '${DOCKERCMD}' not found/broken."
-      bugsystem_finalreport 1
-      cleanup_and_exit 1
-    fi
-  fi
-
-  dockvers=$(docker_version Client)
-  if [[ "${dockvers}" =~ ^0
-     || "${dockvers}" =~ ^1\.[0-5]$ || "${dockvers}" =~ ^1\.[0-5]\. ]]; then
-    if [[ "${DOCKERFAIL}" =~ ^12
-       || "${DOCKERFAIL}" =~ ^2 ]]; then
-      add_vote_table 0 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 1.6.0). Disabling docker."
-      DOCKERSUPPORT=false
-    else
-      add_vote_table -1 docker "Docker command '${DOCKERCMD}' is too old (${dockvers} < 1.6.0). Disabling docker."
-      bugsystem_finalreport 1
-      cleanup_and_exit 1
-    fi
-  fi
-}
-
-## @description  Verify dockerfile exists
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       exits on failure if configured
-function docker_fileverify
-{
-  if [[ ${DOCKERMODE} = false &&
-        ${DOCKERSUPPORT} = true ]]; then
-    if [[ -n "${DOCKERFILE}" ]]; then
-      pushd "${STARTINGDIR}" >/dev/null
-      if [[ -f ${DOCKERFILE} ]]; then
-        DOCKERFILE=$(yetus_abs "${DOCKERFILE}")
-      else
-        if [[ "${DOCKERFAIL}" =~ ^1 ]]; then
-          yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, falling back to built-in."
-          add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, falling back to built-in."
-          DOCKERFILE=${DOCKERFILE_DEFAULT}
-        elif [[ "${DOCKERFAIL}" =~ ^2 ]]; then
-          yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found, disabling docker."
-          add_vote_table 0 docker "Dockerfile '${DOCKERFILE}' not found, disabling docker."
-          DOCKERSUPPORT=false
-        else
-          yetus_error "ERROR: Dockerfile '${DOCKERFILE}' not found."
-          add_vote_table -1 docker "Dockerfile '${DOCKERFILE}' not found."
-          bugsystem_finalreport 1
-          cleanup_and_exit 1
-        fi
-      fi
-      popd >/dev/null
-    else
-      DOCKERFILE=${DOCKERFILE_DEFAULT}
-    fi
-  fi
-}
-
-## @description  Verify docker exists
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @return       1 if docker is broken
-## @return       0 if docker is working
-function docker_exeverify
-{
-  if ! verify_command "Docker" "${DOCKERCMD}"; then
-    return 1
-  fi
-
-  if ! ${DOCKERCMD} info >/dev/null 2>&1; then
-    yetus_error "Docker is not functioning properly. Daemon down/unreachable?"
-    return 1
-  fi
-  return 0
-}
-
-## @description  Run docker with some arguments, and
-## @description  optionally send to debug.
-## @description  some destructive commands require
-## @description  DOCKER_DESTRUCTIVE to be set to true
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        args
-function dockercmd
-{
-  declare subcmd=$1
-  shift
-
-  yetus_debug "dockercmd: ${DOCKERCMD} ${subcmd} $*"
-  if [[ ${subcmd} == rm
-      || ${subcmd} == rmi
-      || ${subcmd} == stop
-      || ${subcmd} == kill ]]; then
-    if [[ "${DOCKER_DESTRUCTIVE}" == false ]]; then
-      yetus_error "Safemode: not running ${DOCKERCMD} ${subcmd} $*"
-      return
-    fi
-  fi
-  "${DOCKERCMD}" "${subcmd}" "$@"
-}
-
-## @description  Convet docker's time format to ctime
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        time
-function dockerdate_to_ctime
-{
-  declare mytime=$1
-
-  if [[ "${mytime}" = "0001-01-01T00:00:00Z" ]]; then
-    mytime="1970-01-01T00:00:00"
-  fi
-
-  # believe it or not, date is not even close to standardized...
-  if [[ $(uname -s) == Linux ]]; then
-
-    # GNU date
-    date -d "${mytime}" "+%s"
-  else
-
-    # BSD date; docker gives us two different format because fun
-    if ! date -j -f "%FT%T%z" "${mytime}" "+%s" 2>/dev/null; then
-      date -j -f "%FT%T" "${mytime}" "+%s"
-    fi
-  fi
-}
-
-## @description  Stop and delete all defunct containers
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function docker_container_maintenance
-{
-  declare line
-  declare id
-  declare name
-  declare status
-  declare tmptime
-  declare createtime
-  declare starttime
-  declare stoptime
-  declare remove
-  declare difftime
-  declare data
-
-  if [[ "${ROBOT}" = false ]]; then
-    return
-  fi
-
-  big_console_header "Docker Container Maintenance"
-
-  dockercmd ps -a
-
-  data=$(dockercmd ps -qa)
-
-  if [[ -z "${data}" ]]; then
-    return
-  fi
-
-  while read -r line; do
-    id=$(echo "${line}" | cut -f1 -d, )
-    status=$(echo "${line}" | cut -f2 -d, )
-    tmptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. )
-    createtime=$(dockerdate_to_ctime "${tmptime}")
-    tmptime=$(echo "${line}" | cut -f4 -d, | cut -f1 -d. )
-    starttime=$(dockerdate_to_ctime "${tmptime}")
-    tmptime=$(echo "${line}" | cut -f5 -d, | cut -f1 -d. )
-    stoptime=$(dockerdate_to_ctime "${tmptime}")
-    name=$(echo "${line}" | cut -f6 -d, )
-    curtime=$("${AWK}" 'BEGIN {srand(); print srand()}')
-    remove=false
-
-    case ${status} in
-      created)
-        ((difftime = curtime - createtime))
-        if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[0]} ]]; then
-          remove=true
-        fi
-      ;;
-      stopped)
-        ((difftime = curtime - stoptime))
-        if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[1]} ]]; then
-          remove=true
-        fi
-      ;;
-      exited | dead)
-        ((difftime = curtime - stoptime))
-        if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[2]} ]]; then
-          remove=true
-        fi
-      ;;
-      running)
-        ((difftime = curtime - starttime))
-        if [[ ${difftime} -gt ${DOCKER_CONTAINER_PURGE[3]}
-             && "${SENTINEL}" = true ]]; then
-          remove=true
-          echo "Attempting to kill docker container ${name} [${id}]"
-          dockercmd kill "${id}"
-        fi
-      ;;
-      *)
-      ;;
-    esac
-
-    if [[ "${remove}" == true ]]; then
-      echo "Attempting to remove docker container ${name} [${id}]"
-      dockercmd rm "${id}"
-    fi
-
-  done < <(
-     # shellcheck disable=SC2086
-     dockercmd inspect \
-        --format '{{.Id}},{{.State.Status}},{{.Created}},{{.State.StartedAt}},{{.State.FinishedAt}},{{.Name}}' \
-       ${data})
-}
-
-## @description  Delete images after ${DOCKER_IMAGE_PURGE}
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function docker_image_maintenance_helper
-{
-  declare id
-  declare tmptime
-  declare createtime
-  declare difftime
-  declare name
-
-  if [[ "${ROBOT}" = false ]]; then
-    return
-  fi
-
-  if [[ -z "$*" ]]; then
-    return
-  fi
-
-  for id in "$@"; do
-    tmptime=$(dockercmd inspect --format '{{.Created}}' "${id}" | cut -f1 -d. )
-    createtime=$(dockerdate_to_ctime "${tmptime}")
-    curtime=$(date "+%s")
-
-    ((difftime = curtime - createtime))
-    if [[ ${difftime} -gt ${DOCKER_IMAGE_PURGE} ]]; then
-      echo "Attempting to remove docker image ${id}"
-      dockercmd rmi "${id}"
-    fi
-  done
-}
-
-
-## @description  get sentinel-level docker images
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        args
-function docker_get_sentinel_images
-{
-  #shellcheck disable=SC2016
-  dockercmd images \
-    | tail -n +2 \
-    | "${GREP}" -v hours \
-    | "${AWK}" '{print $1":"$2}' \
-    | "${GREP}" -v "<none>:<none>"
-}
-
-## @description  Remove untagged/unused images
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        args
-function docker_image_maintenance
-{
-  declare id
-
-  if [[ "${ROBOT}" = false ]]; then
-    return
-  fi
-
-  big_console_header "Removing old images"
-
-  dockercmd images
-
-  echo "Untagged images:"
-
-  #shellcheck disable=SC2046
-  docker_image_maintenance_helper $(dockercmd images --filter "dangling=true" -q --no-trunc)
-
-  echo "Apache Yetus images:"
-
-  # removing this by image id doesn't always work without a force
-  # in the situations that, for whatever reason, docker decided
-  # to use the same image. this was a rare problem with older
-  # releases of yetus. at some point, we should revisit this
-  # in the mean time, we're going to reconstruct the
-  # repostory:tag and send that to get removed.
-
-  #shellcheck disable=SC2046,SC2016
-  docker_image_maintenance_helper $(dockercmd images | ${GREP} -e ^yetus | grep tp- | ${AWK} '{print $1":"$2}')
-  #shellcheck disable=SC2046,SC2016
-  docker_image_maintenance_helper $(dockercmd images | ${GREP} -e ^yetus | ${GREP} -v hours | ${AWK} '{print $1":"$2}')
-
-  if [[ "${SENTINEL}" = false ]]; then
-    return
-  fi
-
-  echo "Other images:"
-  #shellcheck disable=SC2046
-  docker_image_maintenance_helper $(docker_get_sentinel_images)
-}
-
-
-## @description  Perform pre-run maintenance to free up
-## @description  resources. With --jenkins, it is a lot
-## @description  more destructive.
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        args
-function docker_cleanup
-{
-
-  docker_image_maintenance
-
-  docker_container_maintenance
-}
-
-## @description  Determine the revision of a dockerfile
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        args
-function docker_getfilerev
-{
-  ${GREP} 'YETUS_PRIVATE: gitrev=' \
-        "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \
-          | cut -f2 -d=
-}
-
-function docker_version
-{
-  declare vertype=$1
-  declare val
-  declare ret
-
-  # new version command
-  val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null)
-  ret=$?
-
-  if [[ ${ret} != 0 ]];then
-    # old version command
-    val=$(dockercmd version | ${GREP} "${vertype} version" | cut -f2 -d: | tr -d ' ')
-  fi
-
-  echo "${val}"
-}
-
-## @description  Start a test patch docker container
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        args
-function docker_run_image
-{
-  declare dockerfilerev
-  declare baseimagename
-  declare patchimagename="yetus/${PROJECT_NAME}:tp-${DOCKER_ID}"
-  declare containername="yetus_tp-${DOCKER_ID}"
-  declare client
-  declare server
-  declare retval
-  declare elapsed
-
-  dockerfilerev=$(docker_getfilerev)
-
-  baseimagename="yetus/${PROJECT_NAME}:${dockerfilerev}"
-
-  # make a base image, if it isn't available
-  big_console_header "Building base image: ${baseimagename}"
-  start_clock
-  dockercmd build \
-    -t "${baseimagename}" \
-    "${PATCH_DIR}/precommit/test-patch-docker"
-  retval=$?
-
-  #shellcheck disable=SC2046
-  elapsed=$(clock_display $(stop_clock))
-
-  echo ""
-  echo "Total Elapsed time: ${elapsed}"
-  echo ""
-
-  if [[ ${retval} != 0 ]]; then
-    yetus_error "ERROR: Docker failed to build image."
-    add_vote_table -1 docker "Docker failed to build ${baseimagename}."
-    bugsystem_finalreport 1
-    cleanup_and_exit 1
-  fi
-
-  big_console_header "Building ${BUILDMODE} image: ${patchimagename}"
-  start_clock
-  # using the base image, make one that is patch specific
-  dockercmd build \
-    -t "${patchimagename}" \
-    - <<PatchSpecificDocker
-FROM ${baseimagename}
-LABEL org.apache.yetus=""
-LABEL org.apache.yetus.testpatch.patch="tp-${DOCKER_ID}"
-LABEL org.apache.yetus.testpatch.project=${PROJECT_NAME}
-RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME} || true
-RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME} || true
-RUN chown -R ${USER_NAME} /home/${USER_NAME} || true
-ENV HOME /home/${USER_NAME}
-USER ${USER_NAME}
-PatchSpecificDocker
-
-  retval=$?
-
-  #shellcheck disable=SC2046
-  elapsed=$(clock_display $(stop_clock))
-
-  echo ""
-  echo "Total Elapsed time: ${elapsed}"
-  echo ""
-
-  if [[ ${retval} != 0 ]]; then
-    yetus_error "ERROR: Docker failed to build image."
-    add_vote_table -1 docker "Docker failed to build ${patchimagename}."
-    bugsystem_finalreport 1
-    cleanup_and_exit 1
-  fi
-
-  if [[ "${DOCKER_ENABLE_PRIVILEGED}" = true ]]; then
-    DOCKER_EXTRAARGS+=("--privileged")
-  fi
-
-  if [[ -n "${DOCKER_MEMORY}" ]]; then
-    DOCKER_EXTRAARGS+=("-m" "${DOCKER_MEMORY}")
-  fi
-
-  client=$(docker_version Client)
-  server=$(docker_version Server)
-
-  dockerversion="Client=${client} Server=${server}"
-
-
-  # make the kernel prefer to kill us if we run out of RAM
-  DOCKER_EXTRAARGS+=("--oom-score-adj" "500")
-
-  DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile")
-  DOCKER_EXTRAARGS+=(-v "${PWD}:/testptch/${PROJECT_NAME}")
-  DOCKER_EXTRAARGS+=(-u "${USER_NAME}")
-  DOCKER_EXTRAARGS+=(-w "/testptch/${PROJECT_NAME}")
-  DOCKER_EXTRAARGS+=("--env=BASEDIR=/testptch/${PROJECT_NAME}")
-  DOCKER_EXTRAARGS+=("--env=DOCKER_VERSION=${dockerversion} Image:${baseimagename}")
-  DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
-  DOCKER_EXTRAARGS+=("--env=PATCH_SYSTEM=${PATCH_SYSTEM}")
-  DOCKER_EXTRAARGS+=("--env=PROJECT_NAME=${PROJECT_NAME}")
-  DOCKER_EXTRAARGS+=("--env=TESTPATCHMODE=${TESTPATCHMODE}")
-  DOCKER_EXTRAARGS+=(--name "${containername}")
-
-
-  trap 'docker_signal_handler' SIGTERM
-  trap 'docker_signal_handler' SIGINT
-
-  if [[ ${PATCH_DIR} =~ ^/ ]]; then
-    dockercmd run --rm=true -i \
-      "${DOCKER_EXTRAARGS[@]}" \
-      -v "${PATCH_DIR}:/testptch/patchprocess" \
-      --env=PATCH_DIR=/testptch/patchprocess \
-      "${patchimagename}" &
-  else
-    dockercmd run --rm=true -i \
-      "${DOCKER_EXTRAARGS[@]}" \
-      --env=PATCH_DIR="${PATCH_DIR}" \
-      "${patchimagename}" &
-  fi
-
-  wait ${!}
-  cleanup_and_exit $?
-}
-
-## @description  docker kill the container on SIGTERM
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function docker_signal_handler
-{
-  declare cid
-
-  cid=$(cat "${PATCH_DIR}/cidfile")
-
-  yetus_error "ERROR: Caught signal. Killing docker container:"
-  dockercmd kill "${cid}"
-  yetus_error "ERROR: Exiting."
-  cleanup_and_exit 143 # 128 + 15 -- SIGTERM
-}
-
-## @description  Switch over to a Docker container
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        args
-function docker_handler
-{
-  PATCH_DIR=$(relative_dir "${PATCH_DIR}")
-
-  docker_cleanup
-  determine_user
-  docker_run_image
-}


Mime
View raw message