kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [2/3] incubator-kudu git commit: KUDU-1315. Add simple script to build a source release
Date Thu, 28 Jan 2016 02:09:37 GMT
KUDU-1315. Add simple script to build a source release

This adds a new python script meant for interactive use for building
an Apache-style source release.

The source release tarball is simply the output of 'git archive'. The
purpose of the script is to perform some pre-flight checks to make sure
the release seems sane, and to help the user gpg-sign the release upon
completion. It's not fancy, but should ensure that the release process
is repeatable (two committers running this script from the same revision
should result in 100% identical release artifacts).

In order to generate the artifact name, I moved the version string into
a text file at the repository root instead of embedding it into the
CMakeLists.txt file.

Change-Id: Ie5753c4cc822dfbc86210356850ea1b76ab8e7a2
Reviewed-on: http://gerrit.cloudera.org:8080/1930
Reviewed-by: Jean-Daniel Cryans
Tested-by: Todd Lipcon <todd@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/fefa29ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/fefa29ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/fefa29ca

Branch: refs/heads/master
Commit: fefa29cae7698ffc7833138a78228f93a2d1d2ce
Parents: 8325d5d
Author: Todd Lipcon <todd@apache.org>
Authored: Wed Jan 27 14:19:58 2016 -0800
Committer: Todd Lipcon <todd@apache.org>
Committed: Thu Jan 28 00:47:00 2016 +0000

----------------------------------------------------------------------
 CMakeLists.txt                        |   2 +-
 build-support/build_source_release.py | 150 +++++++++++++++++++++++++++++
 build-support/get-upstream-commit.sh  |   4 +-
 build-support/kudu_util.py            |  41 ++++++++
 build-support/push_to_asf.py          |  44 +++------
 version.txt                           |   1 +
 6 files changed, 208 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6bdf7e0..0ac59cf 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,7 +23,7 @@
 cmake_minimum_required(VERSION 3.2.0)
 
 # TODO: can we somehow pass this into the java build?
-set(KUDU_VERSION_NUMBER "0.7.0-SNAPSHOT")
+file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/version.txt" KUDU_VERSION_NUMBER)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 include(CMakeParseArguments)

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/build-support/build_source_release.py
----------------------------------------------------------------------
diff --git a/build-support/build_source_release.py b/build-support/build_source_release.py
new file mode 100755
index 0000000..d6973d9
--- /dev/null
+++ b/build-support/build_source_release.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import hashlib
+import logging
+import os
+import subprocess
+import sys
+
+from kudu_util import check_output, confirm_prompt, Colors, get_my_email
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+GET_UPSTREAM_COMMIT_SCRIPT = os.path.join(ROOT, "build-support", "get-upstream-commit.sh")
+
+
+def check_repo_not_dirty():
+  """Check that the git repository isn't dirty."""
+  dirty_repo = subprocess.call("git diff --quiet && git diff --cached --quiet",
+                               shell=True) != 0
+  if not dirty_repo:
+    return
+  print "The repository does not appear to be clean."
+  print Colors.RED + "The source release will not include your local changes." + \
+      Colors.RESET
+  if not confirm_prompt("Continue?"):
+    sys.exit(1)
+
+
+def check_no_local_commits():
+  """
+  Check that there are no local commits which haven't been pushed to the upstream
+  repo via Jenkins.
+  """
+  upstream_commit = check_output(GET_UPSTREAM_COMMIT_SCRIPT).strip()
+  cur_commit = check_output(["git", "rev-parse", "HEAD"]).strip()
+
+  if upstream_commit == cur_commit:
+    return
+  print "The repository appears to have local commits:"
+  subprocess.check_call(["git", "log", "--oneline", "%s..HEAD" % upstream_commit])
+
+  print Colors.RED + "This should not be an official release!" + \
+      Colors.RESET
+  if not confirm_prompt("Continue?"):
+    sys.exit(1)
+
+
+def get_version_number():
+  """ Return the current version number of Kudu. """
+  return file(os.path.join(ROOT, "version.txt")).read().strip()
+
+
+def create_tarball():
+  artifact_name = "apache-kudu-incubating-%s" % get_version_number()
+  build_dir = os.path.join(ROOT, "build")
+  if not os.path.exists(build_dir):
+    os.path.makedirs(build_dir)
+  tarball_path = os.path.join(build_dir, artifact_name + ".tar.gz")
+  print "Exporting source tarball..."
+  subprocess.check_output(["git", "archive",
+                           "--prefix=%s/" % artifact_name,
+                           "--output=%s" % tarball_path,
+                           "HEAD"])
+  print Colors.GREEN + "Generated tarball:\t" + Colors.RESET, tarball_path
+  return tarball_path
+
+
+def sign_tarball(tarball_path):
+  """ Prompt the user to GPG-sign the tarball using their Apache GPG key. """
+  if not confirm_prompt("Would you like to GPG-sign the tarball now?"):
+    return
+
+  email = get_my_email()
+  if not email.endswith("@apache.org"):
+    print Colors.YELLOW, "Your email address for the repository is not an @apache.org address."
+    print "Release signatures should typically be signed by committers with @apache.org GPG
keys."
+    print Colors.RESET,
+    if not confirm_prompt("Continue?"):
+      return
+
+  try:
+    subprocess.check_call(["gpg", "--detach-sign", "--armor", "-u", email, tarball_path])
+  except subprocess.CalledProcessError:
+    print Colors.RED + "GPG signing failed. Artifact will not be signed." + Colors.RESET
+    return
+  print Colors.GREEN + "Generated signature:\t" + Colors.RESET, tarball_path + ".asc"
+
+
+def checksum_file(summer, path):
+  """
+  Calculates the checksum of the file 'path' using the provided hashlib
+  digest implementation. Returns the hex form of the digest.
+  """
+  with file(path, "rb") as f:
+    # Read the file in 4KB chunks until EOF.
+    for chunk in iter(lambda: f.read(4096), ""):
+      summer.update(chunk)
+  return summer.hexdigest()
+
+
+def gen_checksum_files(tarball_path):
+  """
+  Create md5 and sha files of the tarball.
+
+  The output format is compatible with command line tools like 'sha1sum'
+  and 'md5sum' so they may be used to verify the checksums.
+  """
+  hashes = [(hashlib.sha1, "sha"),
+            (hashlib.md5, "md5")]
+  for hash_func, extension in hashes:
+    digest = checksum_file(hash_func(), tarball_path)
+    path = tarball_path + "." + extension
+    with file(path, "w") as f:
+      print >>f, "%s\t%s" % (digest, os.path.basename(tarball_path))
+    print Colors.GREEN + ("Generated %s:\t" % extension) + Colors.RESET, path
+
+
+def main():
+  # Change into the source repo so that we can run git commands without having to
+  # specify cwd=BUILD_SUPPORT every time.
+  os.chdir(ROOT)
+  check_repo_not_dirty()
+  check_no_local_commits()
+  tarball_path = create_tarball()
+  gen_checksum_files(tarball_path)
+  sign_tarball(tarball_path)
+
+  print Colors.GREEN + "Release successfully generated!" + Colors.RESET
+  print
+
+
+if __name__ == "__main__":
+  logging.basicConfig(level=logging.INFO)
+  main()

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/build-support/get-upstream-commit.sh
----------------------------------------------------------------------
diff --git a/build-support/get-upstream-commit.sh b/build-support/get-upstream-commit.sh
index ecf3a8b..3918c84 100755
--- a/build-support/get-upstream-commit.sh
+++ b/build-support/get-upstream-commit.sh
@@ -21,9 +21,9 @@
 # branch which was checked in by gerrit. This commit hash is printed to
 # stdout.
 #
-# It does so by looking for the verification signoff from Jenkins.  This is
+# It does so by looking for the 'Reviewed-on' tag added by gerrit. This is
 # more foolproof than trying to guess at the "origin/" branch name, since the
 # developer might be working on some local topic branch.
 set -e
 
-git log --grep='Tested-by: \(jenkins\|Internal Jenkins\)' -n1 --pretty=format:%H
+git log --grep='Reviewed-on: ' -n1 --pretty=format:%H

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/build-support/kudu_util.py
----------------------------------------------------------------------
diff --git a/build-support/kudu_util.py b/build-support/kudu_util.py
index 731c1ad..7945da5 100644
--- a/build-support/kudu_util.py
+++ b/build-support/kudu_util.py
@@ -20,7 +20,23 @@
 # This script generates a header file which contains definitions
 # for the current Kudu build (eg timestamp, git hash, etc)
 
+import os
 import subprocess
+import sys
+
+class Colors(object):
+  """ ANSI color codes. """
+
+  def __on_tty(x):
+    if not os.isatty(sys.stdout.fileno()):
+      return ""
+    return x
+
+  RED = __on_tty("\x1b[31m")
+  GREEN = __on_tty("\x1b[32m")
+  YELLOW = __on_tty("\x1b[33m")
+  RESET = __on_tty("\x1b[m")
+
 
 def check_output(*popenargs, **kwargs):
   r"""Run command with arguments and return its output as a byte string.
@@ -39,3 +55,28 @@ def check_output(*popenargs, **kwargs):
     error.output = output
     raise error
   return output
+
+
+def confirm_prompt(prompt):
+  """
+  Issue the given prompt, and ask the user to confirm yes/no. Returns true
+  if the user confirms.
+  """
+  while True:
+    print prompt, "[Y/n]:",
+
+    if not os.isatty(sys.stdout.fileno()):
+      print "Not running interactively. Assuming 'N'."
+      return False
+      pass
+
+    r = raw_input().strip().lower()
+    if r in ['y', 'yes', '']:
+      return True
+    elif r in ['n', 'no']:
+      return False
+
+
+def get_my_email():
+  """ Return the email address in the user's git config. """
+  return check_output(['git', 'config', '--get', 'user.email']).strip()

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/build-support/push_to_asf.py
----------------------------------------------------------------------
diff --git a/build-support/push_to_asf.py b/build-support/push_to_asf.py
index 491830c..0bae118 100755
--- a/build-support/push_to_asf.py
+++ b/build-support/push_to_asf.py
@@ -38,16 +38,16 @@ import re
 import subprocess
 import sys
 
-from kudu_util import check_output
+from kudu_util import check_output, confirm_prompt, Colors, get_my_email
 
 APACHE_REPO = "https://git-wip-us.apache.org/repos/asf/incubator-kudu.git"
 GERRIT_URL_RE = re.compile(r"ssh://.+@gerrit.cloudera.org:29418/kudu")
 
 # ANSI color codes.
-RED = "\x1b[31m"
-GREEN = "\x1b[32m"
-YELLOW = "\x1b[33m"
-RESET = "\x1b[m"
+Colors.RED = "\x1b[31m"
+Colors.GREEN = "\x1b[32m"
+Colors.YELLOW = "\x1b[33m"
+Colors.RESET = "\x1b[m"
 
 # Parsed options, filled in by main().
 OPTIONS = None
@@ -144,24 +144,6 @@ def get_committer_email(rev):
   return check_output(['git', 'log', '-n1', '--pretty=format:%ce', rev]).strip()
 
 
-def get_my_email():
-  """ Return the email address in the user's git config. """
-  return check_output(['git', 'config', '--get', 'user.email']).strip()
-
-
-def confirm_prompt(prompt):
-  """
-  Issue the given prompt, and ask the user to confirm yes/no. Returns true
-  if the user confirms.
-  """
-  while True:
-    print prompt, "[Y/n]:",
-    r = raw_input().strip().lower()
-    if r in ['y', 'yes', '']:
-      return True
-    elif r in ['n', 'no']:
-      return False
-
 def do_update(branch, gerrit_sha, apache_sha):
   """
   Displays and performs a proposed update of the Apache repository
@@ -183,16 +165,16 @@ def do_update(branch, gerrit_sha, apache_sha):
   commits = rev_list("%s..%s" % (apache_sha, gerrit_sha))
   commits.reverse()  # Display from oldest to newest.
   print "-" * 60
-  print GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" % len(commits)) + RESET
+  print Colors.GREEN + ("%d commit(s) need to be pushed from Gerrit to ASF:" % len(commits))
+ Colors.RESET
   push_sha = None
   for sha in commits:
     oneline = describe_commit(sha)
     print "  ", oneline
     committer = get_committer_email(sha)
     if committer != get_my_email():
-      print RED + "   !!! Committed by someone else (%s) !!!" % committer, RESET
+      print Colors.RED + "   !!! Committed by someone else (%s) !!!" % committer, Colors.RESET
       if not confirm_prompt(
-          RED + "   !!! Are you sure you want to push on behalf of another committer?" +
RESET):
+          Colors.RED + "   !!! Are you sure you want to push on behalf of another committer?"
+ Colors.RESET):
         # Even if they don't want to push this commit, we could still push any
         # earlier commits that the user _did_ author.
         if push_sha is not None:
@@ -208,9 +190,9 @@ def do_update(branch, gerrit_sha, apache_sha):
   if OPTIONS.dry_run:
     cmd.append('--dry-run')
   cmd.append('%s:refs/heads/%s' % (push_sha, branch))
-  print GREEN + "Running: " + RESET + " ".join(cmd)
+  print Colors.GREEN + "Running: " + Colors.RESET + " ".join(cmd)
   subprocess.check_call(cmd)
-  print GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + RESET
+  print Colors.GREEN + "Successfully updated %s to %s" % (branch, gerrit_sha) + Colors.RESET
   print
 
 
@@ -240,12 +222,12 @@ def main():
     gerrit_sha = rev_parse("remotes/gerrit/" + branch)
     print "Branch '%s':\t" % branch,
     if gerrit_sha is None:
-      print YELLOW, "found on Apache but not in gerrit", RESET
+      print Colors.YELLOW, "found on Apache but not in gerrit", Colors.RESET
       continue
     if gerrit_sha == apache_sha:
-      print GREEN, "up to date", RESET
+      print Colors.GREEN, "up to date", Colors.RESET
       continue
-    print YELLOW, "needs update", RESET
+    print Colors.YELLOW, "needs update", Colors.RESET
     do_update(branch, gerrit_sha, apache_sha)
 
 

http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/fefa29ca/version.txt
----------------------------------------------------------------------
diff --git a/version.txt b/version.txt
new file mode 100644
index 0000000..978ba2b
--- /dev/null
+++ b/version.txt
@@ -0,0 +1 @@
+0.7.0-SNAPSHOT


Mime
View raw message