drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cgi...@apache.org
Subject [drill] branch master updated: [DRILL-7803] Docker based developer build environment
Date Mon, 16 Nov 2020 17:51:30 GMT
This is an automated email from the ASF dual-hosted git repository.

cgivre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git


The following commit(s) were added to refs/heads/master by this push:
     new 31d6086  [DRILL-7803] Docker based developer build environment
31d6086 is described below

commit 31d6086c4f814c1d7fc476095611e37cc3d95d1c
Author: Niels Basjes <niels@basjes.nl>
AuthorDate: Fri Nov 13 16:37:10 2020 +0100

    [DRILL-7803] Docker based developer build environment
---
 dev-support/docker/Dockerfile          | 155 +++++++++++++++++++++++++++++++++
 dev-support/docker/bashcolors.sh       |  93 ++++++++++++++++++++
 dev-support/docker/drill_env_checks.sh | 121 +++++++++++++++++++++++++
 dev-support/docker/time.sh             |  20 +++++
 docs/dev/Environment.md                |   8 ++
 start-build-env.sh                     | 128 +++++++++++++++++++++++++++
 6 files changed, 525 insertions(+)

diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
new file mode 100644
index 0000000..44b1d45
--- /dev/null
+++ b/dev-support/docker/Dockerfile
@@ -0,0 +1,155 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Dockerfile for installing the necessary dependencies for building Drill.
+# See BUILDING.txt.
+
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_TERSE true
+
+
+# hadolint ignore=DL3008
+RUN apt -q update \
+    && apt install -y software-properties-common apt-utils apt-transport-https \
+    # Repo for different Python versions
+    && add-apt-repository -y ppa:deadsnakes/ppa \
+    && apt-get -q install -y --no-install-recommends \
+        ant \
+        bats \
+        bash-completion \
+        build-essential \
+        bzip2 \
+        ca-certificates \
+        clang \
+        cmake \
+        curl \
+        docker.io \
+        doxygen \
+        findbugs \
+        fuse \
+        g++ \
+        gcc \
+        git \
+        gnupg-agent \
+        libaio1 \
+        libbcprov-java \
+        libbz2-dev \
+        libcurl4-openssl-dev \
+        libfuse-dev \
+        libnuma-dev \
+        libncurses5 \
+        libprotobuf-dev \
+        libprotoc-dev \
+        libsasl2-dev \
+        libsnappy-dev \
+        libssl-dev \
+        libtool \
+        libzstd-dev \
+        locales \
+        make \
+        maven \
+#        openjdk-11-jdk \
+        openjdk-8-jdk \
+        pinentry-curses \
+        pkg-config \
+        python \
+        python2.7 \
+#        python-pip \
+        python-pkg-resources \
+        python-setuptools \
+#        python-wheel \
+        python3-setuptools \
+        python3-pip \
+        python3.5 \
+        python3.6 \
+        python3.7 \
+        python2.7 \
+        virtualenv \
+        tox \
+        rsync \
+        shellcheck \
+        software-properties-common \
+        sudo \
+        valgrind \
+        vim \
+        wget \
+        zlib1g-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+###
+# Install grpcio-tools mypy-protobuf for `python3 sdks/python/setup.py sdist` to work
+###
+RUN pip3 install grpcio-tools mypy-protobuf
+
+###
+# Install Go
+###
+RUN mkdir -p /goroot \
+    && curl https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz | tar xvzf - -C
/goroot --strip-components=1 \
+    && chmod a+rwX -R /goroot
+
+# Set environment variables for Go
+ENV GOROOT /goroot
+ENV GOPATH /gopath
+ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH
+CMD go get github.com/linkedin/goavro
+
+###
+# Miscelaneous fixes...
+###
+# Turns out some build tools use 'time' and in this docker image this is no longer
+# an executable but ONLY an internal command of bash
+COPY time.sh /usr/bin/time
+RUN chmod 755 /usr/bin/time
+
+# Force the complete use of Java 8
+RUN apt remove -y openjdk-11-jre openjdk-11-jre-headless
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+
+
+###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xmx4g -XX:MaxPermSize=512m
+
+###
+# Add a welcome message and environment checks.
+###
+RUN mkdir /scripts
+COPY drill_env_checks.sh /scripts/drill_env_checks.sh
+COPY bashcolors.sh      /scripts/bashcolors.sh
+RUN chmod 755 /scripts /scripts/drill_env_checks.sh /scripts/bashcolors.sh
+
+# hadolint ignore=SC2016
+RUN echo '. /etc/bash_completion'        >> /root/.bash_aliases
+RUN echo '. /scripts/drill_env_checks.sh' >> /root/.bash_aliases
diff --git a/dev-support/docker/bashcolors.sh b/dev-support/docker/bashcolors.sh
new file mode 100755
index 0000000..0f0e05c
--- /dev/null
+++ b/dev-support/docker/bashcolors.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Based upon info on https://wiki.archlinux.org/index.php/Color_Bash_Prompt
+
+# Reset
+export Color_Off='\e[0m'      # Text Reset
+
+# Regular Colors
+export Black='\e[0;30m'       # Black
+export Red='\e[0;31m'         # Red
+export Green='\e[0;32m'       # Green
+export Yellow='\e[0;33m'      # Yellow
+export Blue='\e[0;34m'        # Blue
+export Purple='\e[0;35m'      # Purple
+export Cyan='\e[0;36m'        # Cyan
+export White='\e[0;37m'       # White
+
+# Bold
+export BBlack='\e[1;30m'      # Black
+export BRed='\e[1;31m'        # Red
+export BGreen='\e[1;32m'      # Green
+export BYellow='\e[1;33m'     # Yellow
+export BBlue='\e[1;34m'       # Blue
+export BPurple='\e[1;35m'     # Purple
+export BCyan='\e[1;36m'       # Cyan
+export BWhite='\e[1;37m'      # White
+
+# Underline
+export UBlack='\e[4;30m'      # Black
+export URed='\e[4;31m'        # Red
+export UGreen='\e[4;32m'      # Green
+export UYellow='\e[4;33m'     # Yellow
+export UBlue='\e[4;34m'       # Blue
+export UPurple='\e[4;35m'     # Purple
+export UCyan='\e[4;36m'       # Cyan
+export UWhite='\e[4;37m'      # White
+
+# Background
+export On_Black='\e[40m'      # Black
+export On_Red='\e[41m'        # Red
+export On_Green='\e[42m'      # Green
+export On_Yellow='\e[43m'     # Yellow
+export On_Blue='\e[44m'       # Blue
+export On_Purple='\e[45m'     # Purple
+export On_Cyan='\e[46m'       # Cyan
+export On_White='\e[47m'      # White
+
+# High Intensity
+export IBlack='\e[0;90m'      # Black
+export IRed='\e[0;91m'        # Red
+export IGreen='\e[0;92m'      # Green
+export IYellow='\e[0;93m'     # Yellow
+export IBlue='\e[0;94m'       # Blue
+export IPurple='\e[0;95m'     # Purple
+export ICyan='\e[0;96m'       # Cyan
+export IWhite='\e[0;97m'      # White
+
+# Bold High Intensity
+export BIBlack='\e[1;90m'     # Black
+export BIRed='\e[1;91m'       # Red
+export BIGreen='\e[1;92m'     # Green
+export BIYellow='\e[1;93m'    # Yellow
+export BIBlue='\e[1;94m'      # Blue
+export BIPurple='\e[1;95m'    # Purple
+export BICyan='\e[1;96m'      # Cyan
+export BIWhite='\e[1;97m'     # White
+
+# High Intensity backgrounds
+export On_IBlack='\e[0;100m'  # Black
+export On_IRed='\e[0;101m'    # Red
+export On_IGreen='\e[0;102m'  # Green
+export On_IYellow='\e[0;103m' # Yellow
+export On_IBlue='\e[0;104m'   # Blue
+export On_IPurple='\e[0;105m' # Purple
+export On_ICyan='\e[0;106m'   # Cyan
+export On_IWhite='\e[0;107m'  # White
diff --git a/dev-support/docker/drill_env_checks.sh b/dev-support/docker/drill_env_checks.sh
new file mode 100755
index 0000000..b92182f
--- /dev/null
+++ b/dev-support/docker/drill_env_checks.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# SHELLDOC-IGNORE
+
+# -------------------------------------------------------
+function showWelcome {
+# http://patorjk.com/software/taag/#p=display&f=Doom&t=Drill%20Build%20Env.
+cat << "Welcome-message"
+
+______      _ _ _  ______       _ _     _   _____
+|  _  \    (_) | | | ___ \     (_) |   | | |  ___|
+| | | |_ __ _| | | | |_/ /_   _ _| | __| | | |__ _ ____   __
+| | | | '__| | | | | ___ \ | | | | |/ _` | |  __| '_ \ \ / /
+| |/ /| |  | | | | | |_/ / |_| | | | (_| | | |__| | | \ V /
+|___/ |_|  |_|_|_| \____/ \__,_|_|_|\__,_| \____/_| |_|\_(_)
+
+This is the standard Drill Developer build environment.
+This has all the right tools installed required to build
+Apache Drill from source.
+
+Welcome-message
+}
+
+# -------------------------------------------------------
+
+function showAbort {
+  cat << "Abort-message"
+
+  ___  _                _   _
+ / _ \| |              | | (_)
+/ /_\ \ |__   ___  _ __| |_ _ _ __   __ _
+|  _  | '_ \ / _ \| '__| __| | '_ \ / _\` |
+| | | | |_) | (_) | |  | |_| | | | | (_| |
+\_| |_/_.__/ \___/|_|   \__|_|_| |_|\__, |
+                                     __/ |
+                                    |___/
+
+Abort-message
+}
+
+# -------------------------------------------------------
+
+function failIfUserIsRoot {
+    if [ "$(id -u)" -eq "0" ]; # If you are root then something went wrong.
+    then
+        cat <<End-of-message
+
+Apparently you are inside this docker container as the user root.
+Putting it simply:
+
+   This should not occur.
+
+Known possible causes of this are:
+1) Running this script as the root user ( Just don't )
+2) Running an old docker version ( upgrade to 1.4.1 or higher )
+
+End-of-message
+
+    showAbort
+
+    logout
+
+    fi
+}
+
+# -------------------------------------------------------
+
+function warnIfLowMemory {
+    MINIMAL_MEMORY=2046755
+    INSTALLED_MEMORY=$(grep -F MemTotal /proc/meminfo | awk '{print $2}')
+    if [[ $((INSTALLED_MEMORY)) -lt $((MINIMAL_MEMORY)) ]]; then
+        cat <<End-of-message
+
+ _                    ___  ___
+| |                   |  \\/  |
+| |     _____      __ | .  . | ___ _ __ ___   ___  _ __ _   _
+| |    / _ \\ \\ /\\ / / | |\\/| |/ _ \\ '_ \` _ \\ / _ \\| '__| | | |
+| |___| (_) \\ V  V /  | |  | |  __/ | | | | | (_) | |  | |_| |
+\\_____/\\___/ \\_/\\_/   \\_|  |_/\\___|_| |_| |_|\\___/|_|   \\__, |
+                                                         __/ |
+                                                        |___/
+
+Your system is running on very little memory.
+This means it may work but it wil most likely be slower than needed.
+
+If you are running this via boot2docker you can simply increase
+the available memory to at least ${MINIMAL_MEMORY}KiB
+(you have ${INSTALLED_MEMORY}KiB )
+
+End-of-message
+    fi
+}
+
+# -------------------------------------------------------
+
+showWelcome
+warnIfLowMemory
+failIfUserIsRoot
+
+# -------------------------------------------------------
+
+. "/scripts/bashcolors.sh"
+. "/usr/lib/git-core/git-sh-prompt"
+export PS1='\['${IBlue}${On_Black}'\] \u@\['${IWhite}${On_Red}'\][Drill Build Env.]\['${IBlue}${On_Black}'\]:\['${Cyan}${On_Black}'\]\w$(declare
-F __git_ps1 &>/dev/null && __git_ps1 " \['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\]
]\['${Color_Off}'\]\n$ '
diff --git a/dev-support/docker/time.sh b/dev-support/docker/time.sh
new file mode 100644
index 0000000..94af172
--- /dev/null
+++ b/dev-support/docker/time.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+time "$@"
diff --git a/docs/dev/Environment.md b/docs/dev/Environment.md
index 47d37c1..4660082 100644
--- a/docs/dev/Environment.md
+++ b/docs/dev/Environment.md
@@ -6,6 +6,14 @@ Currently, the Apache Drill build process is known to work on Linux, Windows
and
   * Java 8
   * Maven 3.6.3 or greater
 
+## Docker based build environment
+
+The `start-build-env.sh` script in the root of the project source builds and starts a preconfigured
environment
+that contains all the tools needed to build Apache Drill from source.
+
+This is known to work on Ubuntu 20.04 with Docker installed.
+On other systems your success may vary. On Redhat/CentOS based systems no longer have Docker.
+
 ## Confirm settings
     # java -version
     java version "1.8.0_161"
diff --git a/start-build-env.sh b/start-build-env.sh
new file mode 100755
index 0000000..0740149
--- /dev/null
+++ b/start-build-env.sh
@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e               # exit on error
+
+cd "$(dirname "$0")" # connect to root
+
+DOCKER_DIR=dev-support/docker
+DOCKER_FILE="${DOCKER_DIR}/Dockerfile"
+
+CONTAINER_NAME=drill-dev-${USER}-$$
+
+#CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1)
+#if [ "$CPU_ARCH" = "aarch64" ]; then
+#  DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64"
+#fi
+
+docker build -t drill-build -f $DOCKER_FILE $DOCKER_DIR
+
+USER_NAME=${SUDO_USER:=$USER}
+USER_ID=$(id -u "${USER_NAME}")
+
+if [ "$(uname -s)" = "Darwin" ]; then
+  GROUP_ID=100
+fi
+
+if [ "$(uname -s)" = "Linux" ]; then
+  GROUP_ID=$(id -g "${USER_NAME}")
+  # man docker-run
+  # When using SELinux, mounted directories may not be accessible
+  # to the container. To work around this, with Docker prior to 1.7
+  # one needs to run the "chcon -Rt svirt_sandbox_file_t" command on
+  # the directories. With Docker 1.7 and later the z mount option
+  # does this automatically.
+  if command -v selinuxenabled >/dev/null && selinuxenabled; then
+    DCKR_VER=$(docker -v|
+    awk '$1 == "Docker" && $2 == "version" {split($3,ver,".");print ver[1]"."ver[2]}')
+    DCKR_MAJ=${DCKR_VER%.*}
+    DCKR_MIN=${DCKR_VER#*.}
+    if [ "${DCKR_MAJ}" -eq 1 ] && [ "${DCKR_MIN}" -ge 7 ] ||
+        [ "${DCKR_MAJ}" -gt 1 ]; then
+      V_OPTS=:z
+    else
+      for d in "${PWD}" "${HOME}/.m2"; do
+        ctx=$(stat --printf='%C' "$d"|cut -d':' -f3)
+        if [ "$ctx" != svirt_sandbox_file_t ] && [ "$ctx" != container_file_t ];
then
+          printf 'INFO: SELinux is enabled.\n'
+          printf '\tMounted %s may not be accessible to the container.\n' "$d"
+          printf 'INFO: If so, on the host, run the following command:\n'
+          printf '\t# chcon -Rt svirt_sandbox_file_t %s\n' "$d"
+        fi
+      done
+    fi
+  fi
+fi
+
+# Set the home directory in the Docker container.
+DOCKER_HOME_DIR=${DOCKER_HOME_DIR:-/home/${USER_NAME}}
+
+DOCKER_GROUP_ID=$(getent group docker | cut -d':' -f3)
+
+docker build -t "drill-build-${USER_ID}" - <<UserSpecificDocker
+FROM drill-build
+RUN rm -f /var/log/faillog /var/log/lastlog
+RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
+RUN groupmod -g ${DOCKER_GROUP_ID} docker
+RUN useradd -g ${GROUP_ID} -G docker -u ${USER_ID} -k /root -m ${USER_NAME} -d "${DOCKER_HOME_DIR}"
+RUN echo "${USER_NAME} ALL=NOPASSWD: ALL" > "/etc/sudoers.d/drill-build-${USER_ID}"
+ENV HOME "${DOCKER_HOME_DIR}"
+
+UserSpecificDocker
+
+echo ""
+echo "Docker image build completed."
+echo "=============================================================================================="
+echo ""
+
+#If this env varible is empty, docker will be started
+# in non interactive mode
+DOCKER_INTERACTIVE_RUN=${DOCKER_INTERACTIVE_RUN-"-i -t"}
+
+DOCKER_SOCKET_MOUNT=""
+if [ -S /var/run/docker.sock ];
+then
+  DOCKER_SOCKET_MOUNT="-v /var/run/docker.sock:/var/run/docker.sock${V_OPTS:-}"
+  echo "Enabling Docker support with the docker build environment."
+else
+  echo "There is NO Docker support with the docker build environment."
+fi
+
+COMMAND=( "$@" )
+if [ $# -eq 0 ];
+then
+  COMMAND=( "bash" )
+fi
+
+[ -d "${HOME}/.gradle_drill_build_env" ] || mkdir -p "${HOME}/.gradle_drill_build_env"
+
+# By mapping the .m2 directory you can do an mvn install from
+# within the container and use the result on your normal
+# system.  And this also is a significant speedup in subsequent
+# builds because the dependencies are downloaded only once.
+docker run --rm=true ${DOCKER_INTERACTIVE_RUN}                         \
+           --name "${CONTAINER_NAME}"                                  \
+           -v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}"          \
+           -v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}"    \
+           -v "${HOME}/.gradle_drill_build_env:${DOCKER_HOME_DIR}/.gradle${V_OPTS:-}"  \
+           -v "${PWD}:${DOCKER_HOME_DIR}/drill${V_OPTS:-}"              \
+           -w "${DOCKER_HOME_DIR}/drill"                                \
+           ${DOCKER_SOCKET_MOUNT}                                      \
+           -u "${USER_ID}"                                             \
+           "drill-build-${USER_ID}" "${COMMAND[@]}"


Mime
View raw message