datafu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mha...@apache.org
Subject incubator-datafu git commit: DATAFU-73 Use Apache RAT to verify correct file headers
Date Thu, 27 Nov 2014 15:50:34 GMT
Repository: incubator-datafu
Updated Branches:
  refs/heads/master ad4766482 -> 3eda18951


DATAFU-73 Use Apache RAT to verify correct file headers

https://issues.apache.org/jira/browse/DATAFU-73

Much of this code comes from SAMZA-285.  This does not correct all the missing headers, which
we'll save for another task.


Project: http://git-wip-us.apache.org/repos/asf/incubator-datafu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-datafu/commit/3eda1895
Tree: http://git-wip-us.apache.org/repos/asf/incubator-datafu/tree/3eda1895
Diff: http://git-wip-us.apache.org/repos/asf/incubator-datafu/diff/3eda1895

Branch: refs/heads/master
Commit: 3eda18951b08b002aa5dade393415c9f92578fdb
Parents: ad47664
Author: Matthew Hayes <matthew.terence.hayes@gmail.com>
Authored: Sat Nov 22 16:36:04 2014 -0800
Committer: Matthew Hayes <matthew.terence.hayes@gmail.com>
Committed: Thu Nov 27 07:50:16 2014 -0800

----------------------------------------------------------------------
 .classpath.template                      |  15 --
 .factorypath.template                    |   3 -
 .gitignore                               |   1 +
 .settings/org.eclipse.jdt.apt.core.prefs |   4 -
 .settings/org.eclipse.jdt.core.prefs     |  12 --
 .travis.yml                              |  17 ---
 build.gradle                             |  27 ++++
 datafu-hourglass/.gitignore              |   2 +
 datafu-hourglass/build.gradle            |   7 -
 datafu-pig/build.gradle                  |   7 -
 gradle/buildscript.gradle                |   5 -
 gradle/rat.gradle                        | 109 ++++++++++++++
 gradle/resources/rat-output-to-html.xsl  | 206 ++++++++++++++++++++++++++
 13 files changed, 345 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/.classpath.template
----------------------------------------------------------------------
diff --git a/.classpath.template b/.classpath.template
deleted file mode 100644
index 68d4789..0000000
--- a/.classpath.template
+++ /dev/null
@@ -1,15 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
-	<classpathentry kind="src" path="src/java"/>
-	<classpathentry kind="src" path="test/pig"/>
-	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-	<classpathentry kind="con" path="org.testng.TESTNG_CONTAINER"/>
-	<classpathentry kind="src" path=".apt_generated">
-		<attributes>
-			<attribute name="optional" value="true"/>
-		</attributes>
-	</classpathentry>
-	<classpathentry kind="lib" path="annotation-plugin.jar"/>	
-	<classpathentry kind="con" path="org.apache.ivyde.eclipse.cpcontainer.IVYDE_CONTAINER/?project=@PROJECT@&amp;ivyXmlPath=ivy.xml&amp;confs=*&amp;ivySettingsPath=%24%7Bworkspace_loc%3A@PROJECT@%2Fivysettings.xml%7D&amp;loadSettingsOnDemand=false&amp;propertyFiles=%24%7Bworkspace_loc%3A@PROJECT@%2Fivy%2Flibraries.properties%7D"/>
-	<classpathentry kind="output" path="build/testclasses"/>
-</classpath>

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/.factorypath.template
----------------------------------------------------------------------
diff --git a/.factorypath.template b/.factorypath.template
deleted file mode 100644
index a69b1bf..0000000
--- a/.factorypath.template
+++ /dev/null
@@ -1,3 +0,0 @@
-<factorypath>
-    <factorypathentry kind="WKSPJAR" id="@HOME@/annotation-plugin.jar" enabled="true"
runInBatchMode="false"/>
-</factorypath>

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index bafad11..2b4ada9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,3 +26,4 @@ datafu-pig/input*
 datafu-pig/docs
 datafu-pig/queries
 datafu-pig/query
+*.patch

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/.settings/org.eclipse.jdt.apt.core.prefs
----------------------------------------------------------------------
diff --git a/.settings/org.eclipse.jdt.apt.core.prefs b/.settings/org.eclipse.jdt.apt.core.prefs
deleted file mode 100644
index 7d52ece..0000000
--- a/.settings/org.eclipse.jdt.apt.core.prefs
+++ /dev/null
@@ -1,4 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.apt.aptEnabled=true
-org.eclipse.jdt.apt.genSrcDir=.apt_generated
-org.eclipse.jdt.apt.reconcileEnabled=true

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/.settings/org.eclipse.jdt.core.prefs
----------------------------------------------------------------------
diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 9dec5db..0000000
--- a/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,12 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.processAnnotations=enabled
-org.eclipse.jdt.core.compiler.source=1.6

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 669b97a..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-language: java
-script: ant test -Dtestclasses.pattern=**/datafu/test/pig/$TESTFOLDER/**/*.class
-env:
-  - TESTFOLDER=bags
-  - TESTFOLDER=geo
-  - TESTFOLDER=hash
-  - TESTFOLDER=linkanalysis
-  - TESTFOLDER=random
-  - TESTFOLDER=sampling
-  - TESTFOLDER=sessions
-  - TESTFOLDER=sets
-  - TESTFOLDER=stats
-  - TESTFOLDER=urls
-  - TESTFOLDER=util
-  - TESTFOLDER=text
-jdk:
-  - openjdk6

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index c4fbd29..0869930 100644
--- a/build.gradle
+++ b/build.gradle
@@ -14,6 +14,33 @@ allprojects {
 
 apply from: file("gradle/dependency-versions.gradle")
 apply from: file("gradle/release.gradle")
+apply from: file("gradle/rat.gradle")
+
+rat {
+  excludes = [
+    '**/.git/**',
+    '**/.gradle/**',
+    '**/.project',
+    '**/.factorypath',
+    '**/.settings/**',
+    '**/.classpath',
+    '**/*.iml',
+    '**/*.iws',
+    '**/*.ipr',
+    'gradle/wrapper/**',
+    '.reviewboardrc',
+    'gradlew',
+    'README.md',
+    'RELEASE.md',
+    '**/.DS_Store/**',
+    '**/.gitignore',
+    '**/build/**',
+    '**/target/**',
+    '**/bin/**',
+    '**/test-output/**',
+    'docs/Gemfile.lock'
+  ]
+}
 
 allprojects {
   apply plugin: 'eclipse'

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/datafu-hourglass/.gitignore
----------------------------------------------------------------------
diff --git a/datafu-hourglass/.gitignore b/datafu-hourglass/.gitignore
index b2675dc..942515e 100644
--- a/datafu-hourglass/.gitignore
+++ b/datafu-hourglass/.gitignore
@@ -1,3 +1,5 @@
 test-logs/
 /bin
 /bin
+/target
+/test-output

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/datafu-hourglass/build.gradle
----------------------------------------------------------------------
diff --git a/datafu-hourglass/build.gradle b/datafu-hourglass/build.gradle
index b27b483..19bc0c9 100644
--- a/datafu-hourglass/build.gradle
+++ b/datafu-hourglass/build.gradle
@@ -1,5 +1,4 @@
 apply plugin: 'java'
-apply plugin: 'license'
 
 import groovy.xml.MarkupBuilder
 
@@ -20,12 +19,6 @@ cleanEclipse {
   }
 }
 
-// create tasks to automatically add the license header
-license {
-  header rootProject.file('HEADER')
-  skipExistingHeaders = true
-}
-
 dependencies {
   // core dependencies, listed as dependencies in pom
   compile "log4j:log4j:$log4jVersion"

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/datafu-pig/build.gradle
----------------------------------------------------------------------
diff --git a/datafu-pig/build.gradle b/datafu-pig/build.gradle
index 2e00d40..37535a0 100644
--- a/datafu-pig/build.gradle
+++ b/datafu-pig/build.gradle
@@ -1,5 +1,4 @@
 apply plugin: 'java'
-apply plugin: 'license'
 apply plugin: 'gradle-autojar'
 apply plugin: 'download-task'
 
@@ -123,12 +122,6 @@ artifacts {
   archives finalJar
 }
 
-// create tasks to automatically add the license header
-license {
-  header rootProject.file('HEADER')
-  skipExistingHeaders = true
-}
-
 dependencies {
   // dependencies that are packaged into the jar using autojar
   // autojar only includes what is needed

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/gradle/buildscript.gradle
----------------------------------------------------------------------
diff --git a/gradle/buildscript.gradle b/gradle/buildscript.gradle
index 669eb6e..10b8ca4 100644
--- a/gradle/buildscript.gradle
+++ b/gradle/buildscript.gradle
@@ -23,14 +23,9 @@ repositories {
       // For gradle-nexus-plugin
       url 'http://jcenter.bintray.com/'
     }
-    maven {
-      // For license plugin.
-      url 'http://dl.bintray.com/content/netflixoss/external-gradle-plugins/'
-    }
   }
 }
 
 dependencies {
-  classpath 'nl.javadude.gradle.plugins:license-gradle-plugin:0.6.1'
   classpath 'org.gradle.api.plugins:gradle-nexus-plugin:0.7.1'
 }

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/gradle/rat.gradle
----------------------------------------------------------------------
diff --git a/gradle/rat.gradle b/gradle/rat.gradle
new file mode 100644
index 0000000..8c8ac5e
--- /dev/null
+++ b/gradle/rat.gradle
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.api.internal.project.IsolatedAntBuilder
+
+apply plugin: RatPlugin
+
+class RatTask extends DefaultTask {
+  @Input
+  List<String> excludes
+
+  def reportPath = 'build/rat'
+  def stylesheet = 'gradle/resources/rat-output-to-html.xsl'
+  def xmlReport = reportPath + '/rat-report.xml'
+  def htmlReport = reportPath + '/rat-report.html'
+
+  def generateXmlReport(File reportDir) {
+    def antBuilder = services.get(IsolatedAntBuilder)
+    def ratClasspath = project.configurations.rat
+    antBuilder.withClasspath(ratClasspath).execute {
+      ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml')
+      ant.report(format: 'xml', reportFile: xmlReport) {
+        fileset(dir: ".") {
+          patternset {
+            excludes.each {
+              exclude(name: it)
+            }
+          }
+        }
+      }
+    }
+  }
+
+  def printUnknownFiles() {
+    def ratXml = new XmlParser().parse(xmlReport)
+    ratXml.resource.each { resource ->
+      if (resource.'license-approval'.@name[0] == "false") {
+        println('Unknown license: ' + resource.@name)
+      }
+    }
+  }
+
+  def generateHtmlReport() {
+    def antBuilder = services.get(IsolatedAntBuilder)
+    def ratClasspath = project.configurations.rat
+    antBuilder.withClasspath(ratClasspath).execute {
+      ant.xslt(
+          in: xmlReport,
+          style: stylesheet,
+          out: htmlReport,
+          classpath: ratClasspath)
+    }
+    println('Rat report: ' + htmlReport)
+  }
+
+  @TaskAction
+  def rat() {
+    File reportDir = new File(reportPath)
+    if (!reportDir.exists()) {
+      reportDir.mkdirs()
+    }
+    generateXmlReport(reportDir)
+    printUnknownFiles()
+    generateHtmlReport()
+  }
+}
+
+class RatPlugin implements Plugin<Project> {
+  void apply(Project project) {
+    configureDependencies(project)
+    project.plugins.apply(JavaBasePlugin);
+    Task ratTask = project.task("rat",
+        type: RatTask,
+        group: 'Build',
+        description: 'Runs Apache Rat checks.')
+    project.tasks[JavaBasePlugin.CHECK_TASK_NAME].dependsOn ratTask
+  }
+
+  void configureDependencies(final Project project) {
+    project.configurations {
+      rat
+    }
+    project.repositories {
+      mavenCentral()
+    }
+    project.dependencies {
+      rat 'org.apache.rat:apache-rat-tasks:0.8'
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-datafu/blob/3eda1895/gradle/resources/rat-output-to-html.xsl
----------------------------------------------------------------------
diff --git a/gradle/resources/rat-output-to-html.xsl b/gradle/resources/rat-output-to-html.xsl
new file mode 100644
index 0000000..97ea7a1
--- /dev/null
+++ b/gradle/resources/rat-output-to-html.xsl
@@ -0,0 +1,206 @@
+<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="2.0">
+
+<!--***********************************************************
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ ***********************************************************-->
+
+<!-- This style sheet converts any rat-report.xml file.  -->
+
+<xsl:template match="/">
+
+  <html>
+    <head>
+     <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+     <style type="text/css">
+    &lt;!--
+body {margin-top: 0px;font-size: 0.8em;background-color: #F9F7ED;}
+
+h1 {color:red;}
+h2 {color:blue;}
+h3 {color:green;}
+h4 {color:orange;}
+
+/* Table Design */
+
+table,tr,td {text-align:center;font-weight:bold;border:1px solid #000;}
+caption {color:blue;text-align:left;}
+.notes, .binaries, .archives, .standards {width:25%;}
+.notes {background:#D7EDEE;}
+.binaries {background:#D0F2F4;}
+.archives {background:#ABE7E9;}
+.standards {background:#A0F0F4;}
+.licenced, .generated {width:50%;}
+.licenced {background:#C6EBDD;}
+.generated {background:#ABE9D2;}
+.java_note {background:#D6EBC6;}
+.generated_note {background:#C9E7A9;}
+.unknown {width:100%;background:#E92020;}
+.unknown-zero {color:#00CC00;}
+.center{text-align:center;margin:0 auto;}
+--&gt;
+     </style>
+    </head>
+    <body>
+      <xsl:apply-templates/>
+      <xsl:call-template name="generated"/>
+    </body>
+  </html>
+</xsl:template>
+
+<xsl:template match="rat-report">
+
+  <h1>Rat Report</h1>
+  <p>This HTML version (yes, it is!) is generated from the RAT xml reports using Saxon9B.
All the outputs required are displayed below, similar to the .txt version.
+           This is obviously a work in progress; and a prettier, easier to read and manage
version will be available soon</p>
+<div class="center">
+<table id="rat-reports summary" cellspacing="0" summary="A snapshot summary of this rat
report">
+<caption>
+Table 1: A snapshot summary of this rat report.
+</caption>
+  <tr>
+    <td colspan="1" class="notes">Notes: <xsl:value-of select="count(descendant::type[attribute::name=&quot;notice&quot;])"/></td>
+    <td colspan="1" class="binaries">Binaries: <xsl:value-of select="count(descendant::type[attribute::name=&quot;binary&quot;])"/></td>
+    <td colspan="1" class="archives">Archives: <xsl:value-of select="count(descendant::type[attribute::name=&quot;archive&quot;])"/></td>
+    <td colspan="1" class="standards">Standards: <xsl:value-of select="count(descendant::type[attribute::name=&quot;standard&quot;])"/></td>
+  </tr>
+  <tr>
+    <td colspan="2" class="licenced">Apache Licensed: <xsl:value-of select="count(descendant::header-type[attribute::name=&quot;AL
  &quot;])"/></td>
+    <td colspan="2" class="generated">Generated Documents: <xsl:value-of select="count(descendant::header-type[attribute::name=&quot;GEN
 &quot;])"/></td>
+  </tr>
+  <tr>
+    <td colspan="2" class="java_note">Note: JavaDocs are generated and so license header
is optional</td>
+    <td colspan="2" class="generated_note">Note: Generated files do not require license
headers</td>
+  </tr>
+  <tr>
+<xsl:choose>
+  <xsl:when test="count(descendant::header-type[attribute::name=&quot;?????&quot;])
&gt; 0">
+    <td colspan="4" class="unknown"><xsl:value-of select="count(descendant::header-type[attribute::name=&quot;?????&quot;])"/>
Unknown Licenses - or files without a license.</td>
+  </xsl:when>
+  <xsl:otherwise>
+    <td colspan="4" class="unknown-zero"><xsl:value-of select="count(descendant::header-type[attribute::name=&quot;?????&quot;])"/>
Unknown Licenses - or files without a license.</td>
+  </xsl:otherwise>
+</xsl:choose>
+  </tr>
+</table>
+</div>
+<hr/>
+  <h3>Unapproved Licenses:</h3>
+
+  <xsl:for-each select="descendant::resource[license-approval/@name=&quot;false&quot;]">
+  <xsl:text>  </xsl:text>
+  <xsl:value-of select="@name"/><br/>
+  <xsl:text>
+</xsl:text>
+</xsl:for-each>
+<hr/>
+
+<h3>Archives:</h3>
+
+<xsl:for-each select="descendant::resource[type/@name=&quot;archive&quot;]">
+ + <xsl:value-of select="@name"/>
+ <br/>
+ </xsl:for-each>
+ <hr/>
+
+ <p>
+   Files with Apache License headers will be marked AL<br/>
+   Binary files (which do not require AL headers) will be marked B<br/>
+  Compressed archives will be marked A<br/>
+  Notices, licenses etc will be marked N<br/>
+  </p>
+
+ <xsl:for-each select="descendant::resource">
+  <xsl:choose>
+   <xsl:when test="license-approval/@name=&quot;false&quot;">!</xsl:when>
+   <xsl:otherwise><xsl:text> </xsl:text></xsl:otherwise>
+ </xsl:choose>
+ <xsl:choose>
+   <xsl:when test="type/@name=&quot;notice&quot;">N   </xsl:when>
+   <xsl:when test="type/@name=&quot;archive&quot;">A   </xsl:when>
+   <xsl:when test="type/@name=&quot;binary&quot;">B   </xsl:when>
+   <xsl:when test="type/@name=&quot;standard&quot;"><xsl:value-of select="header-type/@name"/></xsl:when>
+   <xsl:otherwise>!!!!!</xsl:otherwise>
+ </xsl:choose>
+ <xsl:text>      </xsl:text>
+ <xsl:value-of select="@name"/><br/>
+ <xsl:text>
+ </xsl:text>
+ </xsl:for-each>
+ <hr/>
+
+ <h3>Printing headers for files without AL header...</h3>
+
+ <xsl:for-each select="descendant::resource[header-type/@name=&quot;?????&quot;]">
+
+   <h4><xsl:value-of select="@name"/></h4>
+  <xsl:value-of select="header-sample"/>
+  <hr/>
+</xsl:for-each>
+<br/>
+
+ <!-- <xsl:apply-templates select="resource"/>
+    <xsl:apply-templates select="header-sample"/>
+    <xsl:apply-templates select="header-type"/>
+    <xsl:apply-templates select="license-family"/>
+    <xsl:apply-templates select="license-approval"/>
+    <xsl:apply-templates select="type"/> -->
+
+</xsl:template>
+
+<xsl:template match="resource">
+  <div>
+    <h3>Resource: <xsl:value-of select="@name"/></h3>
+      <xsl:apply-templates/>
+    </div>
+</xsl:template>
+
+<xsl:template match="header-sample">
+  <xsl:if test="normalize-space(.) != ''">
+  <h4>First few lines of non-compliant file</h4>
+    <p>
+      <xsl:value-of select="."/>
+    </p>
+    </xsl:if>
+    <h4>Other Info:</h4>
+</xsl:template>
+
+<xsl:template match="header-type">
+  Header Type: <xsl:value-of select="@name"/>
+  <br/>
+</xsl:template>
+
+<xsl:template match="license-family">
+  License Family: <xsl:value-of select="@name"/>
+  <br/>
+</xsl:template>
+
+<xsl:template match="license-approval">
+  License Approval: <xsl:value-of select="@name"/>
+  <br/>
+</xsl:template>
+
+<xsl:template match="type">
+  Type: <xsl:value-of select="@name"/>
+  <br/>
+</xsl:template>
+
+<xsl:template name="generated">
+</xsl:template>
+</xsl:transform>


Mime
View raw message