kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [kudu] 03/03: KUDU-3005: Reduce size of kudu-python docker image
Date Tue, 14 Jan 2020 19:41:35 GMT
This is an automated email from the ASF dual-hosted git repository.

granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit 1650e2f39449a1b89d4c3abba1106b5ad6bf824a
Author: Grant Henke <granthenke@apache.org>
AuthorDate: Wed Dec 18 08:25:11 2019 -0600

    KUDU-3005: Reduce size of kudu-python docker image
    
    This patch breaks out the python environment bootstrapping from
    the dev environment bootstrapping. This allows us to use the
    runtime image for the kudu-python docker image instead of the
    giant dev image.
    
    The result of this change is a 440MB uncompressed image instead
    of a 1.92GB uncompressed image.
    
    I manually tested the image by running the python example against
    a quickstart cluster:
    https://github.com/apache/kudu/blob/master/examples/python/basic-python-example/basic_example.py
    
    Change-Id: I159907c4a85e919d98c92df97cd44d0300d1cd29
    Reviewed-on: http://gerrit.cloudera.org:8080/14984
    Reviewed-by: Adar Dembo <adar@cloudera.com>
    Reviewed-by: Andrew Wong <awong@cloudera.com>
    Tested-by: Kudu Jenkins
---
 .dockerignore                  |  1 +
 docker/Dockerfile              | 21 +++++++---
 docker/bootstrap-dev-env.sh    | 32 ---------------
 docker/bootstrap-python-env.sh | 93 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 110 insertions(+), 37 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 007dd33..1379088 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -30,6 +30,7 @@
 
 # Docker files.
 !docker/bootstrap-dev-env.sh
+!docker/bootstrap-python-env.sh
 !docker/bootstrap-runtime-env.sh
 !docker/kudu-entrypoint.sh
 
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 266c372..444dd32 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -62,7 +62,11 @@ ARG BASE_OS
 FROM $BASE_OS as dev
 
 COPY ./docker/bootstrap-dev-env.sh /
-RUN ./bootstrap-dev-env.sh && rm bootstrap-dev-env.sh
+COPY ./docker/bootstrap-python-env.sh /
+RUN ./bootstrap-dev-env.sh \
+  && ./bootstrap-python-env.sh \
+  && rm bootstrap-dev-env.sh \
+  && rm bootstrap-python-env.sh
 
 # Common label arguments.
 # VCS_REF is not specified to improve docker caching.
@@ -221,13 +225,20 @@ LABEL name="Apache Kudu Build" \
 # ---- Kudu Python ----
 # Builds a runtime image with the Kudu python client pre-installed.
 #
-FROM dev AS kudu-python
+FROM runtime AS kudu-python
+
+COPY ./docker/bootstrap-python-env.sh /
+RUN ./bootstrap-python-env.sh \
+  && rm bootstrap-python-env.sh
+
+ARG INSTALL_DIR="/opt/kudu"
 
 ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib64
 WORKDIR $INSTALL_DIR/python
 # Copy the requirements file.
 COPY ./python/requirements.txt requirements.txt
-COPY --from=build /usr/local /usr/local/
+COPY --from=build /usr/local/lib/libkudu_client* /usr/local/lib/
+COPY --from=build /usr/local/include/kudu /usr/local/include/kudu
 COPY --from=build /kudu/python/dist/kudu-python-*.tar.gz .
 RUN pip install -r requirements.txt \
     && rm -rf requirements.txt \
@@ -241,8 +252,8 @@ ARG VCS_TYPE
 ARG VCS_URL
 ARG VERSION
 
-LABEL org.label-schema.name="Apache Kudu Python Client Base" \
-      org.label-schema.description="Builds a runtime image with the Kudu python client pre-installed."
\
+LABEL org.label-schema.name="Apache Kudu Python Client" \
+      org.label-schema.description="An image with the Kudu Python client pre-installed."
\
       # Common labels.
       org.label-schema.dockerfile=$DOCKERFILE \
       org.label-schema.maintainer=$MAINTAINER \
diff --git a/docker/bootstrap-dev-env.sh b/docker/bootstrap-dev-env.sh
index 4cbddcf..62754ba 100755
--- a/docker/bootstrap-dev-env.sh
+++ b/docker/bootstrap-dev-env.sh
@@ -25,29 +25,6 @@
 
 set -xe
 
-function install_python_packages() {
-  PYTHON_VERSION=$(python --version 2>&1 | cut -d' ' -f2)
-  PYTHON_MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1)
-  PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2)
-
-  # We use get-pip.py to bootstrap pip outside of system packages.
-  # This prevents issues with the platform package manager knowing
-  # about only some of the python packages.
-  if [[ "$PYTHON_MAJOR" == "2" && "$PYTHON_MINOR" == "6" ]]; then
-    # Beginning with pip 10, Python 2.6 is no longer supported.
-    curl https://bootstrap.pypa.io/2.6/get-pip.py | python
-  else
-    # Use a stable version of pip that works with the remaining
-    # versions of Python 2 and 3. pip 19.1 doesn't support Python 3.4,
-    # which is the version of Python 3 shipped with Ubuntu 14.04.
-    curl https://bootstrap.pypa.io/get-pip.py | python - "pip < 19.0"
-  fi
-  pip install --upgrade \
-      cython \
-      setuptools \
-      setuptools_scm
-}
-
 # Install the prerequisite libraries, if they are not installed.
 # CentOS/RHEL
 if [[ -f "/usr/bin/yum" ]]; then
@@ -92,11 +69,6 @@ if [[ -f "/usr/bin/yum" ]]; then
     ruby-devel \
     zlib-devel
 
-  # Install python development packages.
-  yum install -y epel-release
-  yum install -y python-devel
-  install_python_packages
-
   # To build on a version older than 7.0, the Red Hat Developer Toolset
   # must be installed (in order to have access to a C++11 capable compiler).
   OS_MAJOR_VERSION=$(lsb_release -rs | cut -f1 -d.)
@@ -180,10 +152,6 @@ elif [[ -f "/usr/bin/apt-get" ]]; then
     xsltproc \
     zlib1g-dev
 
-  # Install python development packages.
-  apt-get install -y --no-install-recommends python-dev
-  install_python_packages
-
   # Reduce the image size by cleaning up after the install.
   apt-get clean
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
diff --git a/docker/bootstrap-python-env.sh b/docker/bootstrap-python-env.sh
new file mode 100755
index 0000000..ac7f14b
--- /dev/null
+++ b/docker/bootstrap-python-env.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+##########################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This script handles bootstrapping a base OS for
+# the Apache Kudu base docker images.
+#
+##########################################################
+
+set -xe
+
+function install_python_packages() {
+  PYTHON_VERSION=$(python --version 2>&1 | cut -d' ' -f2)
+  PYTHON_MAJOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f1)
+  PYTHON_MINOR=$(echo "$PYTHON_VERSION" | cut -d'.' -f2)
+
+  # We use get-pip.py to bootstrap pip outside of system packages.
+  # This prevents issues with the platform package manager knowing
+  # about only some of the python packages.
+  if [[ "$PYTHON_MAJOR" == "2" && "$PYTHON_MINOR" == "6" ]]; then
+    # Beginning with pip 10, Python 2.6 is no longer supported.
+    curl https://bootstrap.pypa.io/2.6/get-pip.py | python
+  else
+    # Use a stable version of pip that works with the remaining
+    # versions of Python 2 and 3. pip 19.1 doesn't support Python 3.4,
+    # which is the version of Python 3 shipped with Ubuntu 14.04.
+    curl https://bootstrap.pypa.io/get-pip.py | python - "pip < 19.0"
+  fi
+  pip install --upgrade \
+      cython \
+      setuptools \
+      setuptools_scm
+}
+
+# Install the prerequisite libraries, if they are not installed.
+# CentOS/RHEL
+if [[ -f "/usr/bin/yum" ]]; then
+  # Update the repo.
+  yum update -y
+
+  # Install curl, used when installing pip.
+  yum install -y ca-certificates curl
+
+  # Install python development packages.
+  yum install -y epel-release
+  # g++ is required to check for int128 support in setup.py.
+  yum install -y gcc gcc-c++ python-devel
+  install_python_packages
+
+  # Reduce the image size by cleaning up after the install.
+  yum clean all
+  rm -rf /var/cache/yum /tmp/* /var/tmp/*
+# Ubuntu/Debian
+elif [[ -f "/usr/bin/apt-get" ]]; then
+  # Ensure the Debian frontend is noninteractive.
+  export DEBIAN_FRONTEND=noninteractive
+
+  # Update the repo.
+  apt-get update -y
+
+  # Install curl, used when installing pip.
+  apt-get install -y --no-install-recommends ca-certificates curl
+
+  # Install python development packages.
+  # g++ is required to check for int128 support in setup.py.
+  apt-get install -y --no-install-recommends g++ python-dev
+  install_python_packages
+
+  # Reduce the image size by cleaning up after the install.
+  apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+  unset DEBIAN_FRONTEND
+else
+  echo "Unsupported OS"
+  exit 1
+fi


Mime
View raw message