yetus-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a.@apache.org
Subject [02/17] yetus git commit: YETUS-15. build environment
Date Sun, 11 Nov 2018 22:19:18 GMT
http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/src/main/python/releasedocmaker/__init__.py
----------------------------------------------------------------------
diff --git a/releasedocmaker/src/main/python/releasedocmaker/__init__.py b/releasedocmaker/src/main/python/releasedocmaker/__init__.py
new file mode 100755
index 0000000..63476bf
--- /dev/null
+++ b/releasedocmaker/src/main/python/releasedocmaker/__init__.py
@@ -0,0 +1,946 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from glob import glob
+from optparse import OptionParser
+from time import gmtime, strftime, sleep
+from distutils.version import LooseVersion
+import errno
+import os
+import re
+import shutil
+import urllib
+import urllib2
+import httplib
+import json
+sys.dont_write_bytecode = True
+# pylint: disable=wrong-import-position,relative-import
+from utils import get_jira, to_unicode, sanitize_text, processrelnote, Outputs
+# pylint: enable=wrong-import-position
+
+try:
+    import dateutil.parser
+except ImportError:
+    print "This script requires python-dateutil module to be installed. " \
+          "You can install it using:\n\t pip install python-dateutil"
+    sys.exit(1)
+
+RELEASE_VERSION = {}
+
+JIRA_BASE_URL = "https://issues.apache.org/jira"
+SORTTYPE = 'resolutiondate'
+SORTORDER = 'older'
+NUM_RETRIES = 5
+
+# label to be used to mark an issue as Incompatible change.
+BACKWARD_INCOMPATIBLE_LABEL = 'backward-incompatible'
+
+ASF_LICENSE = '''
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+'''
+
+def buildindex(title, asf_license):
+    """Write an index file for later conversion using mvn site"""
+    versions = glob("[0-9]*.[0-9]*")
+    versions.sort(key=LooseVersion, reverse=True)
+    with open("index.md", "w") as indexfile:
+        if asf_license is True:
+            indexfile.write(ASF_LICENSE)
+        for version in versions:
+            indexfile.write("* %s v%s\n" % (title, version))
+            for k in ("Changelog", "Release Notes"):
+                indexfile.write("    * [%s](%s/%s.%s.html)\n" %
+                                (k, version, k.upper().replace(" ", ""),
+                                 version))
+
+
+def buildreadme(title, asf_license):
+    """Write an index file for Github using README.md"""
+    versions = glob("[0-9]*.[0-9]*")
+    versions.sort(key=LooseVersion, reverse=True)
+    with open("README.md", "w") as indexfile:
+        if asf_license is True:
+            indexfile.write(ASF_LICENSE)
+        for version in versions:
+            indexfile.write("* %s v%s\n" % (title, version))
+            for k in ("Changelog", "Release Notes"):
+                indexfile.write("    * [%s](%s/%s.%s.md)\n" %
+                                (k, version, k.upper().replace(" ", ""),
+                                 version))
+
+
+class GetVersions(object):
+    """ List of version strings """
+
+    def __init__(self, versions, projects):
+        versions = versions
+        projects = projects
+        self.newversions = []
+        versions.sort(key=LooseVersion)
+        print "Looking for %s through %s" % (versions[0], versions[-1])
+        newversions = set()
+        for project in projects:
+            url = JIRA_BASE_URL + \
+              "/rest/api/2/project/%s/versions" % project.upper()
+            try:
+                resp = get_jira(url)
+            except (urllib2.HTTPError, urllib2.URLError, httplib.BadStatusLine):
+                sys.exit(1)
+
+            datum = json.loads(resp.read())
+            for data in datum:
+                newversions.add(data['name'])
+        newlist = list(newversions.copy())
+        newlist.append(versions[0])
+        newlist.append(versions[-1])
+        newlist.sort(key=LooseVersion)
+        start_index = newlist.index(versions[0])
+        end_index = len(newlist) - 1 - newlist[::-1].index(versions[-1])
+        for newversion in newlist[start_index + 1:end_index]:
+            if newversion in newversions:
+                print "Adding %s to the list" % newversion
+                self.newversions.append(newversion)
+
+    def getlist(self):
+        return self.newversions
+
+
+class Version(object):
+    """Represents a version number"""
+
+    def __init__(self, data):
+        self.mod = False
+        self.data = data
+        found = re.match(r'^((\d+)(\.\d+)*).*$', data)
+        if found:
+            self.parts = [int(p) for p in found.group(1).split('.')]
+        else:
+            self.parts = []
+        # backfill version with zeros if missing parts
+        self.parts.extend((0,) * (3 - len(self.parts)))
+
+    def __str__(self):
+        if self.mod:
+            return '.'.join([str(p) for p in self.parts])
+        return self.data
+
+    def __cmp__(self, other):
+        return cmp(self.parts, other.parts)
+
+
+class Jira(object):
+    """A single JIRA"""
+
+    def __init__(self, data, parent):
+        self.key = data['key']
+        self.fields = data['fields']
+        self.parent = parent
+        self.notes = None
+        self.incompat = None
+        self.reviewed = None
+        self.important = None
+
+    def get_id(self):
+        return to_unicode(self.key)
+
+    def get_description(self):
+        return to_unicode(self.fields['description'])
+
+    def get_release_note(self):
+        if self.notes is None:
+            field = self.parent.field_id_map['Release Note']
+            if field in self.fields:
+                self.notes = to_unicode(self.fields[field])
+            elif self.get_incompatible_change() or self.get_important():
+                self.notes = self.get_description()
+            else:
+                self.notes = ""
+        return self.notes
+
+    def get_priority(self):
+        ret = ""
+        pri = self.fields['priority']
+        if pri is not None:
+            ret = pri['name']
+        return to_unicode(ret)
+
+    def get_assignee(self):
+        ret = ""
+        mid = self.fields['assignee']
+        if mid is not None:
+            ret = mid['displayName']
+        return to_unicode(ret)
+
+    def get_components(self):
+        if self.fields['components']:
+            return ", ".join([comp['name'] for comp in self.fields['components']
+                             ])
+        return ""
+
+    def get_summary(self):
+        return self.fields['summary']
+
+    def get_type(self):
+        ret = ""
+        mid = self.fields['issuetype']
+        if mid is not None:
+            ret = mid['name']
+        return to_unicode(ret)
+
+    def get_reporter(self):
+        ret = ""
+        mid = self.fields['reporter']
+        if mid is not None:
+            ret = mid['displayName']
+        return to_unicode(ret)
+
+    def get_project(self):
+        ret = ""
+        mid = self.fields['project']
+        if mid is not None:
+            ret = mid['key']
+        return to_unicode(ret)
+
+    def __cmp__(self, other):
+        result = 0
+
+        if SORTTYPE == 'issueid':
+            # compare by issue name-number
+            selfsplit = self.get_id().split('-')
+            othersplit = other.get_id().split('-')
+            result = cmp(selfsplit[0], othersplit[0])
+            if result == 0:
+                result = cmp(int(selfsplit[1]), int(othersplit[1]))
+                # dec is supported for backward compatibility
+                if SORTORDER in ['dec', 'desc']:
+                        result *= -1
+
+        elif SORTTYPE == 'resolutiondate':
+            dts = dateutil.parser.parse(self.fields['resolutiondate'])
+            dto = dateutil.parser.parse(other.fields['resolutiondate'])
+            result = cmp(dts, dto)
+            if SORTORDER == 'newer':
+                    result *= -1
+
+        return result
+
+    def get_incompatible_change(self):
+        if self.incompat is None:
+            field = self.parent.field_id_map['Hadoop Flags']
+            self.reviewed = False
+            self.incompat = False
+            if field in self.fields:
+                if self.fields[field]:
+                    for flag in self.fields[field]:
+                        if flag['value'] == "Incompatible change":
+                            self.incompat = True
+                        if flag['value'] == "Reviewed":
+                            self.reviewed = True
+            else:
+                # Custom field 'Hadoop Flags' is not defined,
+                # search for 'backward-incompatible' label
+                field = self.parent.field_id_map['Labels']
+                if field in self.fields and self.fields[field]:
+                    if BACKWARD_INCOMPATIBLE_LABEL in self.fields[field]:
+                        self.incompat = True
+                        self.reviewed = True
+        return self.incompat
+
+    def get_important(self):
+        if self.important is None:
+            field = self.parent.field_id_map['Flags']
+            self.important = False
+            if field in self.fields:
+                if self.fields[field]:
+                    for flag in self.fields[field]:
+                        if flag['value'] == "Important":
+                            self.important = True
+        return self.important
+
+
+class JiraIter(object):
+    """An Iterator of JIRAs"""
+
+    @staticmethod
+    def collect_fields():
+        """send a query to JIRA and collect field-id map"""
+        try:
+            resp = get_jira(JIRA_BASE_URL + "/rest/api/2/field")
+            data = json.loads(resp.read())
+        except (urllib2.HTTPError, urllib2.URLError, httplib.BadStatusLine, ValueError):
+            sys.exit(1)
+        field_id_map = {}
+        for part in data:
+            field_id_map[part['name']] = part['id']
+        return field_id_map
+
+    @staticmethod
+    def query_jira(ver, projects, pos):
+        """send a query to JIRA and collect
+        a certain number of issue information"""
+        count = 100
+        pjs = "','".join(projects)
+        jql = "project in ('%s') and \
+               fixVersion in ('%s') and \
+               resolution = Fixed" % (pjs, ver)
+        params = urllib.urlencode({'jql': jql,
+                                   'startAt': pos,
+                                   'maxResults': count})
+        return JiraIter.load_jira(params, 0)
+
+    @staticmethod
+    def load_jira(params, fail_count):
+        """send query to JIRA and collect with retries"""
+        try:
+            resp = get_jira(JIRA_BASE_URL + "/rest/api/2/search?%s" % params)
+        except (urllib2.URLError, httplib.BadStatusLine) as err:
+            return JiraIter.retry_load(err, params, fail_count)
+
+        try:
+            data = json.loads(resp.read())
+        except httplib.IncompleteRead as err:
+            return JiraIter.retry_load(err, params, fail_count)
+        return data
+
+    @staticmethod
+    def retry_load(err, params, fail_count):
+        """Retry connection up to NUM_RETRIES times."""
+        print(err)
+        fail_count += 1
+        if fail_count <= NUM_RETRIES:
+            print "Connection failed %d times. Retrying." % (fail_count)
+            sleep(1)
+            return JiraIter.load_jira(params, fail_count)
+        else:
+            print "Connection failed %d times. Aborting." % (fail_count)
+            sys.exit(1)
+
+    @staticmethod
+    def collect_jiras(ver, projects):
+        """send queries to JIRA and collect all issues
+        that belongs to given version and projects"""
+        jiras = []
+        pos = 0
+        end = 1
+        while pos < end:
+            data = JiraIter.query_jira(ver, projects, pos)
+            if 'error_messages' in data:
+                print "JIRA returns error message: %s" % data['error_messages']
+                sys.exit(1)
+            pos = data['startAt'] + data['maxResults']
+            end = data['total']
+            jiras.extend(data['issues'])
+
+            if ver not in RELEASE_VERSION:
+                for issue in data['issues']:
+                    for fix_version in issue['fields']['fixVersions']:
+                        if 'releaseDate' in fix_version:
+                            RELEASE_VERSION[fix_version['name']] = fix_version[
+                                'releaseDate']
+        return jiras
+
+    def __init__(self, version, projects):
+        self.version = version
+        self.projects = projects
+        self.field_id_map = JiraIter.collect_fields()
+        ver = str(version).replace("-SNAPSHOT", "")
+        self.jiras = JiraIter.collect_jiras(ver, projects)
+        self.iter = self.jiras.__iter__()
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        data = self.iter.next()
+        j = Jira(data, self)
+        return j
+
+
+class Linter(object):
+    """Encapsulates lint-related functionality.
+    Maintains running lint statistics about JIRAs."""
+
+    _valid_filters = ["incompatible", "important", "version", "component",
+                      "assignee"]
+
+    def __init__(self, version, options):
+        self._warning_count = 0
+        self._error_count = 0
+        self._lint_message = ""
+        self._version = version
+
+        self._filters = dict(zip(self._valid_filters, [False] * len(
+            self._valid_filters)))
+
+        self.enabled = False
+        self._parse_options(options)
+
+    @staticmethod
+    def add_parser_options(parser):
+        """Add Linter options to passed optparse parser."""
+        filter_string = ", ".join("'" + f + "'" for f in Linter._valid_filters)
+        parser.add_option(
+            "-n",
+            "--lint",
+            dest="lint",
+            action="append",
+            type="string",
+            help="Specify lint filters. Valid filters are " + filter_string +
+            ". " + "'all' enables all lint filters. " +
+            "Multiple filters can be specified comma-delimited and " +
+            "filters can be negated, e.g. 'all,-component'.")
+
+    def _parse_options(self, options):
+        """Parse options from optparse."""
+
+        if options.lint is None or not options.lint:
+            return
+        self.enabled = True
+
+        # Valid filter specifications are
+        # self._valid_filters, negations, and "all"
+        valid_list = self._valid_filters
+        valid_list += ["-" + v for v in valid_list]
+        valid_list += ["all"]
+        valid = set(valid_list)
+
+        enabled = []
+        disabled = []
+
+        for o in options.lint:
+            for token in o.split(","):
+                if token not in valid:
+                    print "Unknown lint filter '%s', valid options are: %s" % \
+                            (token, ", ".join(v for v in sorted(valid)))
+                    sys.exit(1)
+                if token.startswith("-"):
+                    disabled.append(token[1:])
+                else:
+                    enabled.append(token)
+
+        for e in enabled:
+            if e == "all":
+                for f in self._valid_filters:
+                    self._filters[f] = True
+            else:
+                self._filters[e] = True
+        for d in disabled:
+            self._filters[d] = False
+
+    def had_errors(self):
+        """Returns True if a lint error was encountered, else False."""
+        return self._error_count > 0
+
+    def message(self):
+        """Return summary lint message suitable for printing to stdout."""
+        if not self.enabled:
+            return None
+        return self._lint_message + \
+               "\n=======================================" + \
+               "\n%s: Error:%d, Warning:%d \n" % \
+               (self._version, self._error_count, self._warning_count)
+
+    def _check_missing_component(self, jira):
+        """Return if JIRA has a 'missing component' lint error."""
+        if not self._filters["component"]:
+            return False
+
+        if jira.fields['components']:
+            return False
+        return True
+
+    def _check_missing_assignee(self, jira):
+        """Return if JIRA has a 'missing assignee' lint error."""
+        if not self._filters["assignee"]:
+            return False
+
+        if jira.fields['assignee'] is not None:
+            return False
+        return True
+
+    def _check_version_string(self, jira):
+        """Return if JIRA has a version string lint error."""
+        if not self._filters["version"]:
+            return False
+
+        field = jira.parent.field_id_map['Fix Version/s']
+        for ver in jira.fields[field]:
+            found = re.match(r'^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', ver['name'])
+            if not found:
+                return True
+        return False
+
+    def lint(self, jira):
+        """Run lint check on a JIRA."""
+        if not self.enabled:
+            return
+        if not jira.get_release_note():
+            if self._filters["incompatible"] and jira.get_incompatible_change():
+                self._warning_count += 1
+                self._lint_message += "\nWARNING: incompatible change %s lacks release notes." % \
+                                (sanitize_text(jira.get_id()))
+            if self._filters["important"] and jira.get_important():
+                self._warning_count += 1
+                self._lint_message += "\nWARNING: important issue %s lacks release notes." % \
+                                (sanitize_text(jira.get_id()))
+
+        if self._check_version_string(jira):
+            self._warning_count += 1
+            self._lint_message += "\nWARNING: Version string problem for %s " % jira.get_id(
+            )
+
+        if self._check_missing_component(jira) or self._check_missing_assignee(
+                jira):
+            self._error_count += 1
+            error_message = []
+            if self._check_missing_component(jira):
+                error_message.append("component")
+            if self._check_missing_assignee(jira):
+                error_message.append("assignee")
+            self._lint_message += "\nERROR: missing %s for %s " \
+                            % (" and ".join(error_message), jira.get_id())
+
+
+def parse_args():
+    """Parse command-line arguments with optparse."""
+    usage = "usage: %prog [OPTIONS] " + \
+            "--project PROJECT [--project PROJECT] " + \
+            "--version VERSION [--version VERSION2 ...]"
+    parser = OptionParser(
+        usage=usage,
+        epilog=
+        "Markdown-formatted CHANGELOG and RELEASENOTES files will be stored"
+        " in a directory named after the highest version provided.")
+    parser.add_option("--dirversions",
+                      dest="versiondirs",
+                      action="store_true",
+                      default=False,
+                      help="Put files in versioned directories")
+    parser.add_option("--fileversions",
+                      dest="versionfiles",
+                      action="store_true",
+                      default=False,
+                      help="Write files with embedded versions")
+    parser.add_option("-i",
+                      "--index",
+                      dest="index",
+                      action="store_true",
+                      default=False,
+                      help="build an index file")
+    parser.add_option("-l",
+                      "--license",
+                      dest="license",
+                      action="store_true",
+                      default=False,
+                      help="Add an ASF license")
+    parser.add_option("-p",
+                      "--project",
+                      dest="projects",
+                      action="append",
+                      type="string",
+                      help="projects in JIRA to include in releasenotes",
+                      metavar="PROJECT")
+    parser.add_option("-r",
+                      "--range",
+                      dest="range",
+                      action="store_true",
+                      default=False,
+                      help="Given versions are a range")
+    parser.add_option(
+        "--sortorder",
+        dest="sortorder",
+        metavar="TYPE",
+        default=SORTORDER,
+        # dec is supported for backward compatibility
+        choices=["asc", "dec", "desc", "newer", "older"],
+        help="Sorting order for sort type (default: %s)" % SORTORDER)
+    parser.add_option("--sorttype",
+                      dest="sorttype",
+                      metavar="TYPE",
+                      default=SORTTYPE,
+                      choices=["resolutiondate", "issueid"],
+                      help="Sorting type for issues (default: %s)" % SORTTYPE)
+    parser.add_option(
+        "-t",
+        "--projecttitle",
+        dest="title",
+        type="string",
+        help="Title to use for the project (default is Apache PROJECT)")
+    parser.add_option("-u",
+                      "--usetoday",
+                      dest="usetoday",
+                      action="store_true",
+                      default=False,
+                      help="use current date for unreleased versions")
+    parser.add_option("-v",
+                      "--version",
+                      dest="versions",
+                      action="append",
+                      type="string",
+                      help="versions in JIRA to include in releasenotes",
+                      metavar="VERSION")
+    parser.add_option(
+        "-V",
+        dest="release_version",
+        action="store_true",
+        default=False,
+        help="display version information for releasedocmaker and exit.")
+    parser.add_option("-O",
+                      "--outputdir",
+                      dest="output_directory",
+                      action="append",
+                      type="string",
+                      help="specify output directory to put release docs to.")
+    parser.add_option("-B",
+                      "--baseurl",
+                      dest="base_url",
+                      action="append",
+                      type="string",
+                      help="specify base URL of the JIRA instance.")
+    parser.add_option(
+        "--retries",
+        dest="retries",
+        action="append",
+        type="int",
+        help="Specify how many times to retry connection for each URL.")
+    parser.add_option(
+        "--skip-credits",
+        dest="skip_credits",
+        action="store_true",
+        default=False,
+        help="While creating release notes skip the 'reporter' and 'contributor' columns")
+    parser.add_option("-X",
+                      "--incompatiblelabel",
+                      dest="incompatible_label",
+                      default="backward-incompatible",
+                      type="string",
+                      help="Specify the label to indicate backward incompatibility.")
+
+    Linter.add_parser_options(parser)
+
+    if len(sys.argv) <= 1:
+        parser.print_help()
+        sys.exit(1)
+
+    (options, _) = parser.parse_args()
+
+    # Handle the version string right away and exit
+    if options.release_version:
+        with open(
+                os.path.join(
+                    os.path.dirname(__file__), "../VERSION"), 'r') as ver_file:
+            print ver_file.read()
+        sys.exit(0)
+
+    # Validate options
+    if not options.release_version:
+        if options.versions is None:
+            parser.error("At least one version needs to be supplied")
+        if options.projects is None:
+            parser.error("At least one project needs to be supplied")
+        if options.base_url is not None:
+            if len(options.base_url) > 1:
+                parser.error("Only one base URL should be given")
+            else:
+                options.base_url = options.base_url[0]
+        if options.output_directory is not None:
+            if len(options.output_directory) > 1:
+                parser.error("Only one output directory should be given")
+            else:
+                options.output_directory = options.output_directory[0]
+
+    if options.range or len(options.versions) > 1:
+      if not options.versiondirs and not options.versionfiles:
+        parser.error("Multiple versions require either --fileversions or --dirversions")
+
+    return options
+
+
+def main():
+    options = parse_args()
+
+    if options.output_directory is not None:
+        # Create the output directory if it does not exist.
+        try:
+            os.makedirs(options.output_directory)
+        except OSError as exc:
+            if exc.errno == errno.EEXIST and os.path.isdir(
+                    options.output_directory):
+                pass
+            else:
+                print "Unable to create output directory %s: %s" % \
+                        (options.output_directory, exc.message)
+                sys.exit(1)
+        os.chdir(options.output_directory)
+
+    if options.base_url is not None:
+        global JIRA_BASE_URL
+        JIRA_BASE_URL = options.base_url
+
+    if options.incompatible_label is not None:
+        global BACKWARD_INCOMPATIBLE_LABEL
+        BACKWARD_INCOMPATIBLE_LABEL = options.incompatible_label
+
+
+    projects = options.projects
+
+    if options.range is True:
+        versions = [Version(v)
+                    for v in GetVersions(options.versions, projects).getlist()]
+    else:
+        versions = [Version(v) for v in options.versions]
+    versions.sort()
+
+    global SORTTYPE
+    SORTTYPE = options.sorttype
+    global SORTORDER
+    SORTORDER = options.sortorder
+
+    if options.title is None:
+        title = projects[0]
+    else:
+        title = options.title
+
+    if options.retries is not None:
+        global NUM_RETRIES
+        NUM_RETRIES = options.retries[0]
+
+    haderrors = False
+
+    for version in versions:
+        vstr = str(version)
+        linter = Linter(vstr, options)
+        jlist = sorted(JiraIter(vstr, projects))
+        if not jlist:
+            print "There is no issue which has the specified version: %s" % version
+            continue
+
+        if vstr in RELEASE_VERSION:
+            reldate = RELEASE_VERSION[vstr]
+        elif options.usetoday:
+            reldate = strftime("%Y-%m-%d", gmtime())
+        else:
+            reldate = "Unreleased (as of %s)" % strftime("%Y-%m-%d", gmtime())
+
+        if not os.path.exists(vstr) and options.versiondirs:
+            os.mkdir(vstr)
+
+        if options.versionfiles and options.versiondirs:
+          reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md",
+                               "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", [],
+                               {"ver": version,
+                                "date": reldate,
+                                "title": title})
+          choutputs = Outputs("%(ver)s/CHANGELOG.%(ver)s.md",
+                              "%(ver)s/CHANGELOG.%(key)s.%(ver)s.md", [],
+                              {"ver": version,
+                               "date": reldate,
+                               "title": title})
+        elif options.versiondirs:
+          reloutputs = Outputs("%(ver)s/RELEASENOTES.md",
+                               "%(ver)s/RELEASENOTES.%(key)s.md", [],
+                               {"ver": version,
+                                "date": reldate,
+                                "title": title})
+          choutputs = Outputs("%(ver)s/CHANGELOG.md",
+                              "%(ver)s/CHANGELOG.%(key)s.md", [],
+                              {"ver": version,
+                               "date": reldate,
+                               "title": title})
+        elif options.versionfiles:
+          reloutputs = Outputs("RELEASENOTES.%(ver)s.md",
+                               "RELEASENOTES.%(key)s.%(ver)s.md", [],
+                               {"ver": version,
+                                "date": reldate,
+                                "title": title})
+          choutputs = Outputs("CHANGELOG.%(ver)s.md",
+                              "CHANGELOG.%(key)s.%(ver)s.md", [],
+                              {"ver": version,
+                               "date": reldate,
+                               "title": title})
+        else:
+          reloutputs = Outputs("RELEASENOTES.md",
+                               "RELEASENOTES.%(key)s.md", [],
+                               {"ver": version,
+                                "date": reldate,
+                                "title": title})
+          choutputs = Outputs("CHANGELOG.md",
+                              "CHANGELOG.%(key)s.md", [],
+                              {"ver": version,
+                               "date": reldate,
+                               "title": title})
+
+        if options.license is True:
+            reloutputs.write_all(ASF_LICENSE)
+            choutputs.write_all(ASF_LICENSE)
+
+        relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \
+                  'These release notes cover new developer and user-facing ' \
+                  'incompatibilities, important issues, features, and major improvements.\n\n'
+        chhead = '# %(title)s Changelog\n\n' \
+                 '## Release %(ver)s - %(date)s\n'\
+                 '\n'
+
+        reloutputs.write_all(relhead)
+        choutputs.write_all(chhead)
+
+        incompatlist = []
+        importantlist = []
+        buglist = []
+        improvementlist = []
+        newfeaturelist = []
+        subtasklist = []
+        tasklist = []
+        testlist = []
+        otherlist = []
+
+        for jira in jlist:
+            if jira.get_incompatible_change():
+                incompatlist.append(jira)
+            elif jira.get_important():
+                importantlist.append(jira)
+            elif jira.get_type() == "Bug":
+                buglist.append(jira)
+            elif jira.get_type() == "Improvement":
+                improvementlist.append(jira)
+            elif jira.get_type() == "New Feature":
+                newfeaturelist.append(jira)
+            elif jira.get_type() == "Sub-task":
+                subtasklist.append(jira)
+            elif jira.get_type() == "Task":
+                tasklist.append(jira)
+            elif jira.get_type() == "Test":
+                testlist.append(jira)
+            else:
+                otherlist.append(jira)
+
+            line = '* [%s](' % (sanitize_text(jira.get_id())) + JIRA_BASE_URL + \
+                   '/browse/%s) | *%s* | **%s**\n' \
+                   % (sanitize_text(jira.get_id()),
+                      sanitize_text(jira.get_priority()), sanitize_text(jira.get_summary()))
+
+            if jira.get_release_note() or \
+               jira.get_incompatible_change() or jira.get_important():
+                reloutputs.write_key_raw(jira.get_project(), "\n---\n\n")
+                reloutputs.write_key_raw(jira.get_project(), line)
+                if not jira.get_release_note():
+                    line = '\n**WARNING: No release note provided for this change.**\n\n'
+                else:
+                    line = '\n%s\n\n' % (
+                        processrelnote(jira.get_release_note()))
+                reloutputs.write_key_raw(jira.get_project(), line)
+
+            linter.lint(jira)
+
+        if linter.enabled:
+            print linter.message()
+            if linter.had_errors():
+                haderrors = True
+                shutil.rmtree(vstr)
+                continue
+
+        reloutputs.write_all("\n\n")
+        reloutputs.close()
+
+        if options.skip_credits:
+            CHANGEHDR1 = "| JIRA | Summary | Priority | " + \
+                     "Component |\n"
+            CHANGEHDR2 = "|:---- |:---- | :--- |:---- |\n"
+        else:
+            CHANGEHDR1 = "| JIRA | Summary | Priority | " + \
+                         "Component | Reporter | Contributor |\n"
+            CHANGEHDR2 = "|:---- |:---- | :--- |:---- |:---- |:---- |\n"
+
+        if incompatlist:
+            choutputs.write_all("### INCOMPATIBLE CHANGES:\n\n")
+            choutputs.write_all(CHANGEHDR1)
+            choutputs.write_all(CHANGEHDR2)
+            choutputs.write_list(incompatlist, options.skip_credits, JIRA_BASE_URL)
+
+        if importantlist:
+            choutputs.write_all("\n\n### IMPORTANT ISSUES:\n\n")
+            choutputs.write_all(CHANGEHDR1)
+            choutputs.write_all(CHANGEHDR2)
+            choutputs.write_list(importantlist, options.skip_credits, JIRA_BASE_URL)
+
+        if newfeaturelist:
+            choutputs.write_all("\n\n### NEW FEATURES:\n\n")
+            choutputs.write_all(CHANGEHDR1)
+            choutputs.write_all(CHANGEHDR2)
+            choutputs.write_list(newfeaturelist, options.skip_credits, JIRA_BASE_URL)
+
+        if improvementlist:
+            choutputs.write_all("\n\n### IMPROVEMENTS:\n\n")
+            choutputs.write_all(CHANGEHDR1)
+            choutputs.write_all(CHANGEHDR2)
+            choutputs.write_list(improvementlist, options.skip_credits, JIRA_BASE_URL)
+
+        if buglist:
+            choutputs.write_all("\n\n### BUG FIXES:\n\n")
+            choutputs.write_all(CHANGEHDR1)
+            choutputs.write_all(CHANGEHDR2)
+            choutputs.write_list(buglist, options.skip_credits, JIRA_BASE_URL)
+
+        if testlist:
+            choutputs.write_all("\n\n### TESTS:\n\n")
+            choutputs.write_all(CHANGEHDR1)
+            choutputs.write_all(CHANGEHDR2)
+            choutputs.write_list(testlist, options.skip_credits, JIRA_BASE_URL)
+
+        if subtasklist:
+            choutputs.write_all("\n\n### SUB-TASKS:\n\n")
+            choutputs.write_all(CHANGEHDR1)
+            choutputs.write_all(CHANGEHDR2)
+            choutputs.write_list(subtasklist, options.skip_credits, JIRA_BASE_URL)
+
+        if tasklist or otherlist:
+            choutputs.write_all("\n\n### OTHER:\n\n")
+            choutputs.write_all(CHANGEHDR1)
+            choutputs.write_all(CHANGEHDR2)
+            choutputs.write_list(otherlist, options.skip_credits, JIRA_BASE_URL)
+            choutputs.write_list(tasklist, options.skip_credits, JIRA_BASE_URL)
+
+        choutputs.write_all("\n\n")
+        choutputs.close()
+
+    if options.index:
+        buildindex(title, options.license)
+        buildreadme(title, options.license)
+
+    if haderrors is True:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/src/main/python/releasedocmaker/utils.py
----------------------------------------------------------------------
diff --git a/releasedocmaker/src/main/python/releasedocmaker/utils.py b/releasedocmaker/src/main/python/releasedocmaker/utils.py
new file mode 100644
index 0000000..db957d3
--- /dev/null
+++ b/releasedocmaker/src/main/python/releasedocmaker/utils.py
@@ -0,0 +1,201 @@
+#!/usr/bin/env python2
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import os
+import re
+import urllib2
+import sys
+import json
+import httplib
+sys.dont_write_bytecode = True
+
+NAME_PATTERN = re.compile(r' \([0-9]+\)')
+
+def clean(input_string):
+    return sanitize_markdown(re.sub(NAME_PATTERN, "", input_string))
+
+
+def get_jira(jira_url):
+    """ Provide standard method for fetching content from apache jira and
+        handling of potential errors. Returns urllib2 response or
+        raises one of several exceptions."""
+
+    username = os.environ.get('RDM_JIRA_USERNAME')
+    password = os.environ.get('RDM_JIRA_PASSWORD')
+
+    req = urllib2.Request(jira_url)
+    if username and password:
+        basicauth = base64.encodestring("%s:%s" % (username, password)).replace('\n', '')
+        req.add_header('Authorization', 'Basic %s' % basicauth)
+
+    try:
+        response = urllib2.urlopen(req)
+    except urllib2.HTTPError as http_err:
+        code = http_err.code
+        print "JIRA returns HTTP error %d: %s. Aborting." % \
+              (code, http_err.msg)
+        error_response = http_err.read()
+        try:
+            error_response = json.loads(error_response)
+            print "- Please ensure that specified authentication, projects,"\
+                  " fixVersions etc. are correct."
+            for message in error_response['errorMessages']:
+                print "-", message
+        except ValueError:
+            print "FATAL: Could not parse json response from server."
+        sys.exit(1)
+    except urllib2.URLError as url_err:
+        print "Error contacting JIRA: %s\n" % jira_url
+        print "Reason: %s" % url_err.reason
+        raise url_err
+    except httplib.BadStatusLine as err:
+        raise err
+    return response
+
+
+def format_components(input_string):
+    input_string = re.sub(NAME_PATTERN, '', input_string).replace("'", "")
+    if input_string != "":
+        ret = input_string
+    else:
+        # some markdown parsers don't like empty tables
+        ret = "."
+    return clean(ret)
+
+
+# Return the string encoded as UTF-8.
+#
+# This is necessary for handling markdown in Python.
+def encode_utf8(input_string):
+    return input_string.encode('utf-8')
+
+
+# Sanitize Markdown input so it can be handled by Python.
+#
+# The expectation is that the input is already valid Markdown,
+# so no additional escaping is required.
+def sanitize_markdown(input_string):
+    input_string = encode_utf8(input_string)
+    input_string = input_string.replace("\r", "")
+    input_string = input_string.rstrip()
+    return input_string
+
+
+# Sanitize arbitrary text so it can be embedded in MultiMarkdown output.
+#
+# Note that MultiMarkdown is not Markdown, and cannot be parsed as such.
+# For instance, when using pandoc, invoke it as `pandoc -f markdown_mmd`.
+#
+# Calls sanitize_markdown at the end as a final pass.
+def sanitize_text(input_string):
+    escapes = dict()
+    # See: https://daringfireball.net/projects/markdown/syntax#backslash
+    # We only escape a subset of special characters. We ignore characters
+    # that only have significance at the start of a line.
+    slash_escapes = "_<>*|"
+    slash_escapes += "`"
+    slash_escapes += "\\"
+    all_chars = set()
+    # Construct a set of escapes
+    for c in slash_escapes:
+        all_chars.add(c)
+    for c in all_chars:
+        escapes[c] = "\\" + c
+
+    # Build the output string character by character to prevent double escaping
+    output_string = ""
+    for c in input_string:
+        o = c
+        if c in escapes:
+            o = escapes[c]
+        output_string += o
+
+    return sanitize_markdown(output_string.rstrip())
+
+
+# if release notes have a special marker,
+# we'll treat them as already in markdown format
+def processrelnote(input_string):
+    relnote_pattern = re.compile('^\<\!\-\- ([a-z]+) \-\-\>')
+    fmt = relnote_pattern.match(input_string)
+    if fmt is None:
+        return sanitize_text(input_string)
+    return {
+        'markdown': sanitize_markdown(input_string),
+    }.get(fmt.group(1), sanitize_text(input_string))
+
+
+def to_unicode(obj):
+    if obj is None:
+        return ""
+    return unicode(obj)
+
+
+class Outputs(object):
+    """Several different files to output to at the same time"""
+
+    def __init__(self, base_file_name, file_name_pattern, keys, params=None):
+        if params is None:
+            params = {}
+        self.params = params
+        self.base = open(base_file_name % params, 'w')
+        self.others = {}
+        for key in keys:
+            both = dict(params)
+            both['key'] = key
+            self.others[key] = open(file_name_pattern % both, 'w')
+
+    def write_all(self, pattern):
+        both = dict(self.params)
+        both['key'] = ''
+        self.base.write(pattern % both)
+        for key in self.others:
+            both = dict(self.params)
+            both['key'] = key
+            self.others[key].write(pattern % both)
+
+    def write_key_raw(self, key, input_string):
+        self.base.write(input_string)
+        if key in self.others:
+            self.others[key].write(input_string)
+
+    def close(self):
+        self.base.close()
+        for value in self.others.values():
+            value.close()
+
+    def write_list(self, mylist, skip_credits, base_url):
+        """ Take a Jira object and write out the relevants parts in a multimarkdown table line"""
+        for jira in sorted(mylist):
+            if skip_credits:
+                line = '| [{id}]({base_url}/browse/{id}) | {summary} |  ' \
+                       '{priority} | {component} |\n'
+            else:
+                line = '| [{id}]({base_url}/browse/{id}) | {summary} |  ' \
+                       '{priority} | {component} | {reporter} | {assignee} |\n'
+            args = {'id': encode_utf8(jira.get_id()),
+                    'base_url': base_url,
+                    'summary': sanitize_text(jira.get_summary()),
+                    'priority': sanitize_text(jira.get_priority()),
+                    'component': format_components(jira.get_components()),
+                    'reporter': sanitize_text(jira.get_reporter()),
+                    'assignee': sanitize_text(jira.get_assignee())
+                   }
+            line = line.format(**args)
+            self.write_key_raw(jira.get_project(), line)

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/releasedocmaker/src/main/shell/releasedocmaker
----------------------------------------------------------------------
diff --git a/releasedocmaker/src/main/shell/releasedocmaker b/releasedocmaker/src/main/shell/releasedocmaker
new file mode 100755
index 0000000..30b13b1
--- /dev/null
+++ b/releasedocmaker/src/main/shell/releasedocmaker
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Make a special version of the shell wrapper for releasedocmaker
+# that maintains the ability to have '--lint' mean '--lint=all'
+
+args=()
+for arg in "${@}"; do
+  if [ "${arg}" = "-n" ] || [ "${arg}" = "--lint" ]; then
+    args=("${args[@]}" "--lint=all")
+  else
+    args=("${args[@]}" "${arg}")
+  fi
+done
+
+exec "$(dirname -- "${BASH_SOURCE-0}")/../lib/releasedocmaker/releasedocmaker.py" "${args[@]}"

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/pom.xml
----------------------------------------------------------------------
diff --git a/shelldocs/pom.xml b/shelldocs/pom.xml
new file mode 100644
index 0000000..c5c3ec3
--- /dev/null
+++ b/shelldocs/pom.xml
@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+                      http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.yetus</groupId>
+    <artifactId>yetus-project</artifactId>
+    <version>0.9.0-SNAPSHOT</version>
+    <relativePath>..</relativePath>
+  </parent>
+  <artifactId>shelldocs</artifactId>
+  <description>API Documentation for Shell Scripts</description>
+  <name>Apache Yetus - shelldocs</name>
+  <packaging>jar</packaging>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.scijava</groupId>
+      <artifactId>jython-shaded</artifactId>
+      <version>${jython-shaded.version}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+
+  <resources>
+    <resource>
+      <directory>src/main/python</directory>
+      </resource>
+    </resources>
+
+    <plugins>
+
+      <plugin>
+        <groupId>net.sf.mavenjython</groupId>
+        <artifactId>jython-compile-maven-plugin</artifactId>
+        <version>${jython-compile-maven-plugin.version}</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>jython</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.yetus</groupId>
+            <artifactId>yetus-assemblies</artifactId>
+            <version>${project.version}</version>
+          </dependency>
+        </dependencies>
+
+        <executions>
+
+          <execution>
+            <id>make-jar-assembly</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+            <configuration>
+              <archive>
+                <manifest>
+                  <mainClass>org.apache.yetus.shelldocs.ShellDocs</mainClass>
+                </manifest>
+              </archive>
+              <descriptorRefs>
+                <descriptorRef>jar-with-dependencies</descriptorRef>
+              </descriptorRefs>
+            </configuration>
+          </execution>
+
+          <execution>
+            <id>build</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+            <configuration>
+              <finalName>dist/apache-yetus-${project.version}</finalName>
+              <appendAssemblyId>false</appendAssemblyId>
+              <attach>false</attach>
+              <descriptorRefs>
+                <descriptorRef>script-bundle</descriptorRef>
+              </descriptorRefs>
+            </configuration>
+          </execution>
+
+          <execution>
+            <id>module-dist</id>
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+            <configuration>
+              <appendAssemblyId>false</appendAssemblyId>
+              <attach>true</attach>
+              <descriptorRefs>
+                <descriptorRef>module-dist</descriptorRef>
+              </descriptorRefs>
+            </configuration>
+          </execution>
+
+        </executions>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.yetus</groupId>
+        <artifactId>yetus-minimaven-plugin</artifactId>
+        <version>${project.version}</version>
+        <executions>
+          <execution>
+            <id>bins4libs</id>
+            <phase>prepare-package</phase>
+            <goals>
+              <goal>bin4libs</goal>
+            </goals>
+            <configuration>
+              <libdir>lib/shelldocs</libdir>
+              <basedir>${project.build.directory}/dist/apache-yetus-${project.version}</basedir>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
+      <plugin>
+        <artifactId>maven-deploy-plugin</artifactId>
+        <configuration>
+          <skip>true</skip>
+        </configuration>
+      </plugin>
+
+    </plugins>
+  </build>
+
+</project>

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/shelldocs.py
----------------------------------------------------------------------
diff --git a/shelldocs/shelldocs.py b/shelldocs/shelldocs.py
deleted file mode 100755
index a096cbe..0000000
--- a/shelldocs/shelldocs.py
+++ /dev/null
@@ -1,426 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Do this immediately to prevent compiled forms
-import sys
-import os
-import re
-from optparse import OptionParser
-
-sys.dont_write_bytecode = True
-
-ASFLICENSE = '''
-<!---
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
--->
-'''
-
-
-def docstrip(key, dstr):
-    '''remove extra spaces from shelldoc phrase'''
-    dstr = re.sub("^## @%s " % key, "", dstr)
-    dstr = dstr.lstrip()
-    dstr = dstr.rstrip()
-    return dstr
-
-
-def toc(tlist):
-    '''build a table of contents'''
-    tocout = []
-    header = ()
-    for i in tlist:
-        if header != i.getinter():
-            header = i.getinter()
-            line = "  * %s\n" % (i.headerbuild())
-            tocout.append(line)
-        line = "    * [%s](#%s)\n" % (i.getname().replace("_", r"\_"),
-                                      i.getname())
-        tocout.append(line)
-    return tocout
-
-
-class ShellFunction(object):
-    """a shell function"""
-
-    def __init__(self, filename):
-        '''Initializer'''
-        self.name = None
-        self.audience = None
-        self.stability = None
-        self.replaceb = None
-        self.returnt = None
-        self.desc = None
-        self.params = None
-        self.filename = filename
-        self.linenum = 0
-
-    def __cmp__(self, other):
-        '''comparison'''
-        if self.audience == other.audience:
-            if self.stability == other.stability:
-                if self.replaceb == other.replaceb:
-                    return cmp(self.name, other.name)
-                else:
-                    if self.replaceb == "Yes":
-                        return -1
-            else:
-                if self.stability == "Stable":
-                    return -1
-        else:
-            if self.audience == "Public":
-                return -1
-        return 1
-
-    def reset(self):
-        '''empties current function'''
-        self.name = None
-        self.audience = None
-        self.stability = None
-        self.replaceb = None
-        self.returnt = None
-        self.desc = None
-        self.params = None
-        self.linenum = 0
-        self.filename = None
-
-    def getfilename(self):
-        '''get the name of the function'''
-        if self.filename is None:
-            return "undefined"
-        return self.filename
-
-    def setname(self, text):
-        '''set the name of the function'''
-        definition = text.split()
-        self.name = definition[1]
-
-    def getname(self):
-        '''get the name of the function'''
-        if self.name is None:
-            return "None"
-        return self.name
-
-    def setlinenum(self, linenum):
-        '''set the line number of the function'''
-        self.linenum = linenum
-
-    def getlinenum(self):
-        '''get the line number of the function'''
-        return self.linenum
-
-    def setaudience(self, text):
-        '''set the audience of the function'''
-        self.audience = docstrip("audience", text)
-        self.audience = self.audience.capitalize()
-
-    def getaudience(self):
-        '''get the audience of the function'''
-        if self.audience is None:
-            return "None"
-        return self.audience
-
-    def setstability(self, text):
-        '''set the stability of the function'''
-        self.stability = docstrip("stability", text)
-        self.stability = self.stability.capitalize()
-
-    def getstability(self):
-        '''get the stability of the function'''
-        if self.stability is None:
-            return "None"
-        return self.stability
-
-    def setreplace(self, text):
-        '''set the replacement state'''
-        self.replaceb = docstrip("replaceable", text)
-        self.replaceb = self.replaceb.capitalize()
-
-    def getreplace(self):
-        '''get the replacement state'''
-        if self.replaceb == "Yes":
-            return self.replaceb
-        return "No"
-
-    def getinter(self):
-        '''get the function state'''
-        return self.getaudience(), self.getstability(), self.getreplace()
-
-    def addreturn(self, text):
-        '''add a return state'''
-        if self.returnt is None:
-            self.returnt = []
-        self.returnt.append(docstrip("return", text))
-
-    def getreturn(self):
-        '''get the complete return state'''
-        if self.returnt is None:
-            return "Nothing"
-        return "\n\n".join(self.returnt)
-
-    def adddesc(self, text):
-        '''add to the description'''
-        if self.desc is None:
-            self.desc = []
-        self.desc.append(docstrip("description", text))
-
-    def getdesc(self):
-        '''get the description'''
-        if self.desc is None:
-            return "None"
-        return " ".join(self.desc)
-
-    def addparam(self, text):
-        '''add a parameter'''
-        if self.params is None:
-            self.params = []
-        self.params.append(docstrip("param", text))
-
-    def getparams(self):
-        '''get all of the parameters'''
-        if self.params is None:
-            return ""
-        return " ".join(self.params)
-
-    def getusage(self):
-        '''get the usage string'''
-        line = "%s %s" % (self.name, self.getparams())
-        return line.rstrip()
-
-    def headerbuild(self):
-        '''get the header for this function'''
-        if self.getreplace() == "Yes":
-            replacetext = "Replaceable"
-        else:
-            replacetext = "Not Replaceable"
-        line = "%s/%s/%s" % (self.getaudience(), self.getstability(),
-                             replacetext)
-        return line
-
-    def getdocpage(self):
-        '''get the built document page for this function'''
-        line = "### `%s`\n\n"\
-             "* Synopsis\n\n"\
-             "```\n%s\n"\
-             "```\n\n" \
-             "* Description\n\n" \
-             "%s\n\n" \
-             "* Returns\n\n" \
-             "%s\n\n" \
-             "| Classification | Level |\n" \
-             "| :--- | :--- |\n" \
-             "| Audience | %s |\n" \
-             "| Stability | %s |\n" \
-             "| Replaceable | %s |\n\n" \
-             % (self.getname(),
-                self.getusage(),
-                self.getdesc(),
-                self.getreturn(),
-                self.getaudience(),
-                self.getstability(),
-                self.getreplace())
-        return line
-
-    def lint(self):
-        '''Lint this function'''
-        getfuncs = {
-            "audience": self.getaudience,
-            "stability": self.getstability,
-            "replaceable": self.getreplace,
-        }
-        validvalues = {
-            "audience": ("Public", "Private"),
-            "stability": ("Stable", "Evolving"),
-            "replaceable": ("Yes", "No"),
-        }
-        messages = []
-        for attr in ("audience", "stability", "replaceable"):
-            value = getfuncs[attr]()
-            if value == "None":
-                messages.append("%s:%u: ERROR: function %s has no @%s" %
-                                (self.getfilename(), self.getlinenum(),
-                                 self.getname(), attr.lower()))
-            elif value not in validvalues[attr]:
-                validvalue = "|".join(v.lower() for v in validvalues[attr])
-                messages.append(
-                    "%s:%u: ERROR: function %s has invalid value (%s) for @%s (%s)"
-                    % (self.getfilename(), self.getlinenum(), self.getname(),
-                       value.lower(), attr.lower(), validvalue))
-        return "\n".join(messages)
-
-    def __str__(self):
-        '''Generate a string for this function'''
-        line = "{%s %s %s %s}" \
-          % (self.getname(),
-             self.getaudience(),
-             self.getstability(),
-             self.getreplace())
-        return line
-
-
-def marked_as_ignored(file_path):
-    """Checks for the presence of the marker(SHELLDOC-IGNORE) to ignore the file.
-
-    Marker needs to be in a line of its own and can not
-    be an inline comment.
-
-    A leading '#' and white-spaces(leading or trailing)
-    are trimmed before checking equality.
-
-    Comparison is case sensitive and the comment must be in
-    UPPERCASE.
-    """
-    with open(file_path) as input_file:
-        for line_num, line in enumerate(input_file, 1):
-            if line.startswith("#") and line[1:].strip() == "SHELLDOC-IGNORE":
-                print >> sys.stderr, "Yo! Got an ignore directive in",\
-                                    "file:{} on line number:{}".format(file_path, line_num)
-                return True
-        return False
-
-
-def main():
-    '''main entry point'''
-    parser = OptionParser(
-        usage="usage: %prog [--skipprnorep] " + "[--output OUTFILE|--lint] " +
-        "--input INFILE " + "[--input INFILE ...]",
-        epilog=
-        "You can mark a file to be ignored by shelldocs by adding"
-        " 'SHELLDOC-IGNORE' as comment in its own line."
-        )
-    parser.add_option("-o",
-                      "--output",
-                      dest="outfile",
-                      action="store",
-                      type="string",
-                      help="file to create",
-                      metavar="OUTFILE")
-    parser.add_option("-i",
-                      "--input",
-                      dest="infile",
-                      action="append",
-                      type="string",
-                      help="file to read",
-                      metavar="INFILE")
-    parser.add_option("--skipprnorep",
-                      dest="skipprnorep",
-                      action="store_true",
-                      help="Skip Private & Not Replaceable")
-    parser.add_option("--lint",
-                      dest="lint",
-                      action="store_true",
-                      help="Enable lint mode")
-    parser.add_option(
-        "-V",
-        "--version",
-        dest="release_version",
-        action="store_true",
-        default=False,
-        help="display version information for shelldocs and exit.")
-
-    (options, dummy_args) = parser.parse_args()
-
-    if options.release_version:
-        with open(
-                os.path.join(
-                    os.path.dirname(__file__), "../VERSION"), 'r') as ver_file:
-            print ver_file.read()
-        sys.exit(0)
-
-    if options.infile is None:
-        parser.error("At least one input file needs to be supplied")
-    elif options.outfile is None and options.lint is None:
-        parser.error(
-            "At least one of output file and lint mode needs to be specified")
-
-    allfuncs = []
-    try:
-        for filename in options.infile:
-            with open(filename, "r") as shellcode:
-                # if the file contains a comment containing
-                # only "SHELLDOC-IGNORE" then skip that file
-                if marked_as_ignored(filename):
-                    continue
-                funcdef = ShellFunction(filename)
-                linenum = 0
-                for line in shellcode:
-                    linenum = linenum + 1
-                    if line.startswith('## @description'):
-                        funcdef.adddesc(line)
-                    elif line.startswith('## @audience'):
-                        funcdef.setaudience(line)
-                    elif line.startswith('## @stability'):
-                        funcdef.setstability(line)
-                    elif line.startswith('## @replaceable'):
-                        funcdef.setreplace(line)
-                    elif line.startswith('## @param'):
-                        funcdef.addparam(line)
-                    elif line.startswith('## @return'):
-                        funcdef.addreturn(line)
-                    elif line.startswith('function'):
-                        funcdef.setname(line)
-                        funcdef.setlinenum(linenum)
-                        if options.skipprnorep and \
-                          funcdef.getaudience() == "Private" and \
-                          funcdef.getreplace() == "No":
-                            pass
-                        else:
-                            allfuncs.append(funcdef)
-                        funcdef = ShellFunction(filename)
-    except IOError, err:
-        print >> sys.stderr, "ERROR: Failed to read from file: %s. Aborting." % err.filename
-        sys.exit(1)
-
-    allfuncs = sorted(allfuncs)
-
-    if options.lint:
-        for funcs in allfuncs:
-            message = funcs.lint()
-            if message:
-                print message
-
-    if options.outfile is not None:
-        with open(options.outfile, "w") as outfile:
-            outfile.write(ASFLICENSE)
-            for line in toc(allfuncs):
-                outfile.write(line)
-            outfile.write("\n------\n\n")
-
-            header = []
-            for funcs in allfuncs:
-                if header != funcs.getinter():
-                    header = funcs.getinter()
-                    line = "## %s\n" % (funcs.headerbuild())
-                    outfile.write(line)
-                outfile.write(funcs.getdocpage())
-
-
-if __name__ == "__main__":
-    main()

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/src/main/java/org/apache/yetus/shelldocs/ShellDocs.java
----------------------------------------------------------------------
diff --git a/shelldocs/src/main/java/org/apache/yetus/shelldocs/ShellDocs.java b/shelldocs/src/main/java/org/apache/yetus/shelldocs/ShellDocs.java
new file mode 100644
index 0000000..facbc48
--- /dev/null
+++ b/shelldocs/src/main/java/org/apache/yetus/shelldocs/ShellDocs.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.yetus.shelldocs;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.python.core.Py;
+import org.python.core.PyException;
+import org.python.core.PySystemState;
+import org.python.util.PythonInterpreter;
+
+public class ShellDocs {
+  public static void main(final String[] args) throws PyException {
+    List<String> list = new LinkedList<String>(Arrays.asList(args));
+    list.add(0,"shelldocs");
+    String[] newargs = list.toArray(new String[list.size()]);
+    PythonInterpreter.initialize(System.getProperties(), System.getProperties(), newargs);
+    PySystemState systemState = Py.getSystemState();
+    PythonInterpreter interpreter = new PythonInterpreter();
+    systemState.__setattr__("_jy_interpreter", Py.java2py(interpreter));
+    String command = "try:\n"
+                   + "  import shelldocs\n"
+                   + "  shelldocs.main()\n"
+                   + "except"
+                   + "  SystemExit: pass";
+    interpreter.exec(command);
+  }
+}

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/src/main/python/shelldocs.py
----------------------------------------------------------------------
diff --git a/shelldocs/src/main/python/shelldocs.py b/shelldocs/src/main/python/shelldocs.py
new file mode 100755
index 0000000..57f912f
--- /dev/null
+++ b/shelldocs/src/main/python/shelldocs.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python2
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+sys.dont_write_bytecode = True
+# pylint: disable=wrong-import-position,import-self
+import shelldocs
+# pylint: enable=wrong-import-position
+# pylint: disable=no-member
+shelldocs.main()

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/shelldocs/src/main/python/shelldocs/__init__.py
----------------------------------------------------------------------
diff --git a/shelldocs/src/main/python/shelldocs/__init__.py b/shelldocs/src/main/python/shelldocs/__init__.py
new file mode 100755
index 0000000..a096cbe
--- /dev/null
+++ b/shelldocs/src/main/python/shelldocs/__init__.py
@@ -0,0 +1,426 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Do this immediately to prevent compiled forms
+import sys
+import os
+import re
+from optparse import OptionParser
+
+sys.dont_write_bytecode = True
+
+ASFLICENSE = '''
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+'''
+
+
+def docstrip(key, dstr):
+    '''remove extra spaces from shelldoc phrase'''
+    dstr = re.sub("^## @%s " % key, "", dstr)
+    dstr = dstr.lstrip()
+    dstr = dstr.rstrip()
+    return dstr
+
+
+def toc(tlist):
+    '''build a table of contents'''
+    tocout = []
+    header = ()
+    for i in tlist:
+        if header != i.getinter():
+            header = i.getinter()
+            line = "  * %s\n" % (i.headerbuild())
+            tocout.append(line)
+        line = "    * [%s](#%s)\n" % (i.getname().replace("_", r"\_"),
+                                      i.getname())
+        tocout.append(line)
+    return tocout
+
+
+class ShellFunction(object):
+    """a shell function"""
+
+    def __init__(self, filename):
+        '''Initializer'''
+        self.name = None
+        self.audience = None
+        self.stability = None
+        self.replaceb = None
+        self.returnt = None
+        self.desc = None
+        self.params = None
+        self.filename = filename
+        self.linenum = 0
+
+    def __cmp__(self, other):
+        '''comparison'''
+        if self.audience == other.audience:
+            if self.stability == other.stability:
+                if self.replaceb == other.replaceb:
+                    return cmp(self.name, other.name)
+                else:
+                    if self.replaceb == "Yes":
+                        return -1
+            else:
+                if self.stability == "Stable":
+                    return -1
+        else:
+            if self.audience == "Public":
+                return -1
+        return 1
+
+    def reset(self):
+        '''empties current function'''
+        self.name = None
+        self.audience = None
+        self.stability = None
+        self.replaceb = None
+        self.returnt = None
+        self.desc = None
+        self.params = None
+        self.linenum = 0
+        self.filename = None
+
+    def getfilename(self):
+        '''get the name of the function'''
+        if self.filename is None:
+            return "undefined"
+        return self.filename
+
+    def setname(self, text):
+        '''set the name of the function'''
+        definition = text.split()
+        self.name = definition[1]
+
+    def getname(self):
+        '''get the name of the function'''
+        if self.name is None:
+            return "None"
+        return self.name
+
+    def setlinenum(self, linenum):
+        '''set the line number of the function'''
+        self.linenum = linenum
+
+    def getlinenum(self):
+        '''get the line number of the function'''
+        return self.linenum
+
+    def setaudience(self, text):
+        '''set the audience of the function'''
+        self.audience = docstrip("audience", text)
+        self.audience = self.audience.capitalize()
+
+    def getaudience(self):
+        '''get the audience of the function'''
+        if self.audience is None:
+            return "None"
+        return self.audience
+
+    def setstability(self, text):
+        '''set the stability of the function'''
+        self.stability = docstrip("stability", text)
+        self.stability = self.stability.capitalize()
+
+    def getstability(self):
+        '''get the stability of the function'''
+        if self.stability is None:
+            return "None"
+        return self.stability
+
+    def setreplace(self, text):
+        '''set the replacement state'''
+        self.replaceb = docstrip("replaceable", text)
+        self.replaceb = self.replaceb.capitalize()
+
+    def getreplace(self):
+        '''get the replacement state'''
+        if self.replaceb == "Yes":
+            return self.replaceb
+        return "No"
+
+    def getinter(self):
+        '''get the function state'''
+        return self.getaudience(), self.getstability(), self.getreplace()
+
+    def addreturn(self, text):
+        '''add a return state'''
+        if self.returnt is None:
+            self.returnt = []
+        self.returnt.append(docstrip("return", text))
+
+    def getreturn(self):
+        '''get the complete return state'''
+        if self.returnt is None:
+            return "Nothing"
+        return "\n\n".join(self.returnt)
+
+    def adddesc(self, text):
+        '''add to the description'''
+        if self.desc is None:
+            self.desc = []
+        self.desc.append(docstrip("description", text))
+
+    def getdesc(self):
+        '''get the description'''
+        if self.desc is None:
+            return "None"
+        return " ".join(self.desc)
+
+    def addparam(self, text):
+        '''add a parameter'''
+        if self.params is None:
+            self.params = []
+        self.params.append(docstrip("param", text))
+
+    def getparams(self):
+        '''get all of the parameters'''
+        if self.params is None:
+            return ""
+        return " ".join(self.params)
+
+    def getusage(self):
+        '''get the usage string'''
+        line = "%s %s" % (self.name, self.getparams())
+        return line.rstrip()
+
+    def headerbuild(self):
+        '''get the header for this function'''
+        if self.getreplace() == "Yes":
+            replacetext = "Replaceable"
+        else:
+            replacetext = "Not Replaceable"
+        line = "%s/%s/%s" % (self.getaudience(), self.getstability(),
+                             replacetext)
+        return line
+
+    def getdocpage(self):
+        '''get the built document page for this function'''
+        line = "### `%s`\n\n"\
+             "* Synopsis\n\n"\
+             "```\n%s\n"\
+             "```\n\n" \
+             "* Description\n\n" \
+             "%s\n\n" \
+             "* Returns\n\n" \
+             "%s\n\n" \
+             "| Classification | Level |\n" \
+             "| :--- | :--- |\n" \
+             "| Audience | %s |\n" \
+             "| Stability | %s |\n" \
+             "| Replaceable | %s |\n\n" \
+             % (self.getname(),
+                self.getusage(),
+                self.getdesc(),
+                self.getreturn(),
+                self.getaudience(),
+                self.getstability(),
+                self.getreplace())
+        return line
+
+    def lint(self):
+        '''Lint this function'''
+        getfuncs = {
+            "audience": self.getaudience,
+            "stability": self.getstability,
+            "replaceable": self.getreplace,
+        }
+        validvalues = {
+            "audience": ("Public", "Private"),
+            "stability": ("Stable", "Evolving"),
+            "replaceable": ("Yes", "No"),
+        }
+        messages = []
+        for attr in ("audience", "stability", "replaceable"):
+            value = getfuncs[attr]()
+            if value == "None":
+                messages.append("%s:%u: ERROR: function %s has no @%s" %
+                                (self.getfilename(), self.getlinenum(),
+                                 self.getname(), attr.lower()))
+            elif value not in validvalues[attr]:
+                validvalue = "|".join(v.lower() for v in validvalues[attr])
+                messages.append(
+                    "%s:%u: ERROR: function %s has invalid value (%s) for @%s (%s)"
+                    % (self.getfilename(), self.getlinenum(), self.getname(),
+                       value.lower(), attr.lower(), validvalue))
+        return "\n".join(messages)
+
+    def __str__(self):
+        '''Generate a string for this function'''
+        line = "{%s %s %s %s}" \
+          % (self.getname(),
+             self.getaudience(),
+             self.getstability(),
+             self.getreplace())
+        return line
+
+
+def marked_as_ignored(file_path):
+    """Checks for the presence of the marker(SHELLDOC-IGNORE) to ignore the file.
+
+    Marker needs to be in a line of its own and can not
+    be an inline comment.
+
+    A leading '#' and white-spaces(leading or trailing)
+    are trimmed before checking equality.
+
+    Comparison is case sensitive and the comment must be in
+    UPPERCASE.
+    """
+    with open(file_path) as input_file:
+        for line_num, line in enumerate(input_file, 1):
+            if line.startswith("#") and line[1:].strip() == "SHELLDOC-IGNORE":
+                print >> sys.stderr, "Yo! Got an ignore directive in",\
+                                    "file:{} on line number:{}".format(file_path, line_num)
+                return True
+        return False
+
+
+def main():
+    '''main entry point'''
+    parser = OptionParser(
+        usage="usage: %prog [--skipprnorep] " + "[--output OUTFILE|--lint] " +
+        "--input INFILE " + "[--input INFILE ...]",
+        epilog=
+        "You can mark a file to be ignored by shelldocs by adding"
+        " 'SHELLDOC-IGNORE' as comment in its own line."
+        )
+    parser.add_option("-o",
+                      "--output",
+                      dest="outfile",
+                      action="store",
+                      type="string",
+                      help="file to create",
+                      metavar="OUTFILE")
+    parser.add_option("-i",
+                      "--input",
+                      dest="infile",
+                      action="append",
+                      type="string",
+                      help="file to read",
+                      metavar="INFILE")
+    parser.add_option("--skipprnorep",
+                      dest="skipprnorep",
+                      action="store_true",
+                      help="Skip Private & Not Replaceable")
+    parser.add_option("--lint",
+                      dest="lint",
+                      action="store_true",
+                      help="Enable lint mode")
+    parser.add_option(
+        "-V",
+        "--version",
+        dest="release_version",
+        action="store_true",
+        default=False,
+        help="display version information for shelldocs and exit.")
+
+    (options, dummy_args) = parser.parse_args()
+
+    if options.release_version:
+        with open(
+                os.path.join(
+                    os.path.dirname(__file__), "../VERSION"), 'r') as ver_file:
+            print ver_file.read()
+        sys.exit(0)
+
+    if options.infile is None:
+        parser.error("At least one input file needs to be supplied")
+    elif options.outfile is None and options.lint is None:
+        parser.error(
+            "At least one of output file and lint mode needs to be specified")
+
+    allfuncs = []
+    try:
+        for filename in options.infile:
+            with open(filename, "r") as shellcode:
+                # if the file contains a comment containing
+                # only "SHELLDOC-IGNORE" then skip that file
+                if marked_as_ignored(filename):
+                    continue
+                funcdef = ShellFunction(filename)
+                linenum = 0
+                for line in shellcode:
+                    linenum = linenum + 1
+                    if line.startswith('## @description'):
+                        funcdef.adddesc(line)
+                    elif line.startswith('## @audience'):
+                        funcdef.setaudience(line)
+                    elif line.startswith('## @stability'):
+                        funcdef.setstability(line)
+                    elif line.startswith('## @replaceable'):
+                        funcdef.setreplace(line)
+                    elif line.startswith('## @param'):
+                        funcdef.addparam(line)
+                    elif line.startswith('## @return'):
+                        funcdef.addreturn(line)
+                    elif line.startswith('function'):
+                        funcdef.setname(line)
+                        funcdef.setlinenum(linenum)
+                        if options.skipprnorep and \
+                          funcdef.getaudience() == "Private" and \
+                          funcdef.getreplace() == "No":
+                            pass
+                        else:
+                            allfuncs.append(funcdef)
+                        funcdef = ShellFunction(filename)
+    except IOError, err:
+        print >> sys.stderr, "ERROR: Failed to read from file: %s. Aborting." % err.filename
+        sys.exit(1)
+
+    allfuncs = sorted(allfuncs)
+
+    if options.lint:
+        for funcs in allfuncs:
+            message = funcs.lint()
+            if message:
+                print message
+
+    if options.outfile is not None:
+        with open(options.outfile, "w") as outfile:
+            outfile.write(ASFLICENSE)
+            for line in toc(allfuncs):
+                outfile.write(line)
+            outfile.write("\n------\n\n")
+
+            header = []
+            for funcs in allfuncs:
+                if header != funcs.getinter():
+                    header = funcs.getinter()
+                    line = "## %s\n" % (funcs.headerbuild())
+                    outfile.write(line)
+                outfile.write(funcs.getdocpage())
+
+
+if __name__ == "__main__":
+    main()

http://git-wip-us.apache.org/repos/asf/yetus/blob/6ebaa111/start-build-env.sh
----------------------------------------------------------------------
diff --git a/start-build-env.sh b/start-build-env.sh
new file mode 100755
index 0000000..e773e4a
--- /dev/null
+++ b/start-build-env.sh
@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e               # exit on error
+ROOTDIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE-$0}")" >/dev/null && pwd -P)
+
+# moving to the path of the Dockerfile reduces the context
+cd "${ROOTDIR}/precommit/src/main/shell/test-patch-docker"
+
+BRANCH=$(git branch | grep '\*' | cut -d ' ' -f2)
+if [[ "${BRANCH}" =~ HEAD ]]; then
+  BRANCH=$(git branch | grep '\*' | awk '{print $NF}'  | sed -e s,rel/,,g -e s,\),,g )
+fi
+
+echo "Attempting a pull of apache/yetus-base:${BRANCH} and apache/yetus-base:latest to save time"
+echo "Errors here will be ignored!"
+docker pull "apache/yetus-base:${BRANCH}" || docker pull "apache/yetus-base:latest" || true
+
+docker build -t "apache/yetus-build:${BRANCH}" .
+
+USER_NAME=${SUDO_USER:=$USER}
+USER_ID=$(id -u "${USER_NAME}")
+GROUP_ID=$(id -g "${USER_NAME}")
+
+# When using SELinux, mounted directories may not be accessible
+# to the container. To work around this, with Docker prior to 1.7
+# one needs to run the "chcon -Rt svirt_sandbox_file_t" command on
+# the directories. With Docker 1.7 and later the z mount option
+# does this automatically.
+if command -v selinuxenabled >/dev/null && selinuxenabled; then
+  DCKR_VER=$(docker -v|
+  awk '$1 == "Docker" && $2 == "version" {split($3,ver,".");print ver[1]"."ver[2]}')
+  DCKR_MAJ=${DCKR_VER%.*}
+  DCKR_MIN=${DCKR_VER#*.}
+  if [[ "${DCKR_MAJ}" -eq 1 ]] && [[ "${DCKR_MIN}" -ge 7 ]] ||
+     [[ "${DCKR_MAJ}" -gt 1 ]]; then
+    V_OPTS=:z
+  else
+    for d in "${PWD}" "${HOME}/.m2"; do
+      ctx=$(stat --printf='%C' "$d"|cut -d':' -f3)
+      if [ "$ctx" != svirt_sandbox_file_t ] && [ "$ctx" != container_file_t ]; then
+        printf 'INFO: SELinux is enabled.\n'
+        printf '\tMounted %s may not be accessible to the container.\n' "$d"
+        printf 'INFO: If so, on the host, run the following command:\n'
+        printf '\t# chcon -Rt svirt_sandbox_file_t %s\n' "$d"
+      fi
+    done
+  fi
+fi
+
+cd "${ROOTDIR}/asf-site-src"
+docker build \
+  -t "apache/yetus-build-${USER_ID}:${BRANCH}" \
+  --build-arg GROUP_ID="${GROUP_ID}" \
+  --build-arg USER_ID="${USER_ID}" \
+  --build-arg USER_NAME="${USER_NAME}" \
+  --build-arg DOCKER_TAG="${BRANCH}" \
+  .
+
+# now cd back
+cd "${ROOTDIR}"
+# By mapping the .m2 directory you can do an mvn install from
+# within the container and use the result on your normal
+# system.  And this also is a significant speedup in subsequent
+# builds because the dependencies are downloaded only once.
+# Additionally, we mount GPG and SSH directories so that
+# release managers can use the container to do releases
+docker run --rm=true -i -t \
+  -v "${PWD}:/home/${USER_NAME}/yetus${V_OPTS:-}" \
+  -w "/home/${USER_NAME}/yetus" \
+  -v "${HOME}/.m2:/home/${USER_NAME}/.m2${V_OPTS:-}" \
+  -v "${HOME}/.gnupg:/home/${USER_NAME}/.gnupg" \
+  -v "${HOME}/.ssh:/home/${USER_NAME}/.ssh" \
+  -u "${USER_NAME}" \
+  "apache/yetus-build-${USER_ID}:${BRANCH}" "$@"


Mime
View raw message