beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Work logged] (BEAM-4495) Create website pre-commits for apache/beam repository
Date Fri, 21 Sep 2018 17:06:00 GMT

     [ https://issues.apache.org/jira/browse/BEAM-4495?focusedWorklogId=146471&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-146471
]

ASF GitHub Bot logged work on BEAM-4495:
----------------------------------------

                Author: ASF GitHub Bot
            Created on: 21/Sep/18 17:05
            Start Date: 21/Sep/18 17:05
    Worklog Time Spent: 10m 
      Work Description: swegner closed pull request #5809: [BEAM-4495] Add Gradle tasks for
website and add Jenkins pre-commit job
URL: https://github.com/apache/beam/pull/5809
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/.test-infra/jenkins/PrecommitJobBuilder.groovy b/.test-infra/jenkins/PrecommitJobBuilder.groovy
index cd4250e33e6..15d371d33fc 100644
--- a/.test-infra/jenkins/PrecommitJobBuilder.groovy
+++ b/.test-infra/jenkins/PrecommitJobBuilder.groovy
@@ -26,6 +26,9 @@ class PrecommitJobBuilder {
   /** Base name for each post-commit suite job, i.e. 'Go'. */
   String nameBase
 
+  /** The Docker image to run within, or blank to not use Docker. */
+  String dockerImage
+
   /**  The Gradle task to execute. */
   String gradleTask
 
@@ -96,7 +99,8 @@ class PrecommitJobBuilder {
       common_job_properties.setTopLevelMainJobProperties(delegate,
       'master',
       timeoutMins,
-      allowRemotePoll) // needed for included regions PR triggering; see [JENKINS-23606]
+      allowRemotePoll, // needed for included regions PR triggering; see [JENKINS-23606]
+      dockerImage)
       steps {
         gradle {
           rootBuildScriptDir(common_job_properties.checkoutDir)
diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy
index 81ec59bee18..15b1c97b381 100644
--- a/.test-infra/jenkins/common_job_properties.groovy
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -62,13 +62,15 @@ class common_job_properties {
   static void setTopLevelMainJobProperties(def context,
                                            String branch = 'master',
                                            int timeout = 100,
-                                           boolean allowRemotePoll = true) {
+                                           boolean allowRemotePoll = true,
+                                           String dockerImage = '') {
     setTopLevelJobProperties(
             context,
             'beam',
             branch,
             timeout,
-            allowRemotePoll)
+            allowRemotePoll,
+            dockerImage)
   }
 
   // Sets common top-level job properties. Accessed through one of the above
@@ -77,7 +79,8 @@ class common_job_properties {
                                                String repositoryName,
                                                String defaultBranch,
                                                int defaultTimeout,
-                                               boolean allowRemotePoll = true) {
+                                               boolean allowRemotePoll = true,
+                                               String dockerImage='') {
     def jenkinsExecutorLabel = 'beam'
 
     // GitHub project.
@@ -109,6 +112,13 @@ class common_job_properties {
     }
 
     context.wrappers {
+      if (dockerImage) {
+        throw new UnsupportedOperationException(
+          '[INFRA-16524] Running jobs within Docker not yet supported.')
+        // buildInDocker {
+        //   image(dockerImage)
+        // }
+      }
       // Abort the build if it's stuck for more minutes than specified.
       timeout {
         absolute(defaultTimeout)
diff --git a/.test-infra/jenkins/job_PreCommit_Website.groovy b/.test-infra/jenkins/job_PreCommit_Website.groovy
new file mode 100644
index 00000000000..4010fc22e21
--- /dev/null
+++ b/.test-infra/jenkins/job_PreCommit_Website.groovy
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PrecommitJobBuilder
+
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+    scope: this,
+    nameBase: 'Website',
+    // TODO(BEAM-4696): Run these within docker so we don't need install step
+    // dockerImage: 'ruby:2.5',
+    gradleTask: ':setupJenkins :websitePreCommit',
+    triggerPathPatterns: ['^website/.*$']
+)
+builder.build()
+
diff --git a/.test-infra/jenkins/job_PreCommit_Website_Merge.groovy b/.test-infra/jenkins/job_PreCommit_Website_Merge.groovy
index 71fa13eb728..547508bd16f 100644
--- a/.test-infra/jenkins/job_PreCommit_Website_Merge.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Website_Merge.groovy
@@ -18,6 +18,9 @@
 
 import common_job_properties
 
+// TODO(BEAM-4505): This job is for the apache/beam-site repository and
+// should be removed once website sources are migrated to apache/beam.
+
 // Defines a job.
 job('beam_PreCommit_Website_Merge') {
   description('Runs website tests for mergebot.')
diff --git a/.test-infra/jenkins/job_PreCommit_Website_Stage.groovy b/.test-infra/jenkins/job_PreCommit_Website_Stage.groovy
index d1a79dac15c..c541ddf8a3c 100644
--- a/.test-infra/jenkins/job_PreCommit_Website_Stage.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Website_Stage.groovy
@@ -18,6 +18,9 @@
 
 import common_job_properties
 
+// TODO(BEAM-4505): This job is for the apache/beam-site repository and
+// should be removed once website sources are migrated to apache/beam.
+
 // Defines a job.
 job('beam_PreCommit_Website_Stage') {
   description('Stages the pull requests proposed for the Apache Beam ' +
diff --git a/.test-infra/jenkins/job_PreCommit_Website_Test.groovy b/.test-infra/jenkins/job_PreCommit_Website_Test.groovy
index dd11871ef02..9540291ffca 100644
--- a/.test-infra/jenkins/job_PreCommit_Website_Test.groovy
+++ b/.test-infra/jenkins/job_PreCommit_Website_Test.groovy
@@ -18,6 +18,9 @@
 
 import common_job_properties
 
+// TODO(BEAM-4505): This job is for the apache/beam-site repository and
+// should be removed once website sources are migrated to apache/beam.
+
 // Defines a job.
 job('beam_PreCommit_Website_Test') {
   description('Runs tests on the pull requests proposed for the Apache Beam ' +
diff --git a/build.gradle b/build.gradle
index cccb4b9c7b4..0780f51a31a 100644
--- a/build.gradle
+++ b/build.gradle
@@ -128,6 +128,9 @@ rat {
     "**/_config.yml",
     "**/_config_test.yml",
     "**/.htaccess",
+    "website/.bundle/**/*",
+    "website/.sass-cache/**/*",
+    "website/.testcontent/**/*",
     "website/src/_sass/_bootstrap.scss",
     "website/src/_sass/bootstrap/**/*",
     "website/src/js/bootstrap*.js",
@@ -210,6 +213,12 @@ task pythonPostCommit() {
   dependsOn ":beam-sdks-python:postCommit"
 }
 
+task websitePreCommit() {
+  dependsOn ":rat"
+  dependsOn ":beam-website:preCommit"
+}
+
+
 task runBeamDependencyCheck() {
   dependsOn ":dependencyUpdates"
   dependsOn ":beam-sdks-python:dependencyUpdates"
diff --git a/settings.gradle b/settings.gradle
index 07ae1adead7..f72643e8e4e 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -164,3 +164,5 @@ include "beam-sdks-python"
 project(":beam-sdks-python").dir = file("sdks/python")
 include "beam-sdks-python-container"
 project(":beam-sdks-python-container").dir = file("sdks/python/container")
+include "beam-website"
+project(":beam-website").dir = file("website")
diff --git a/website/.gitignore b/website/.gitignore
index 818a7304531..072e28bc055 100644
--- a/website/.gitignore
+++ b/website/.gitignore
@@ -3,7 +3,7 @@ _site
 .jekyll-metadata
 vendor/
 .bundle/
-.testcontent/
+content/
 
 # Ignore IntelliJ files.
 .idea/
diff --git a/website/Rakefile b/website/Rakefile
index c2e7069c667..6addceafb09 100644
--- a/website/Rakefile
+++ b/website/Rakefile
@@ -3,9 +3,7 @@ require 'html-proofer'
 require 'etc'
 
 task :test do
-  FileUtils.rm_rf('./.testcontent')
-  sh "bundle exec jekyll build --config _config.yml,_config_test.yml"
-  HTMLProofer.check_directory("./.testcontent", {
+  HTMLProofer.check_directory("./content", {
     :typhoeus => {
       :timeout => 60,
       :connecttimeout => 40 },
@@ -13,6 +11,10 @@ task :test do
     :check_html => true,
     :file_ignore => [/javadoc/, /v2/, /pydoc/],
     :url_ignore => [
+        # Javadocs and Pydocs are only available on asf-site branch
+        /documentation\/sdks\/javadoc/,
+        /documentation\/sdks\/pydoc/,
+
         /jstorm.io/,
         /datatorrent.com/,
         /ai.google/ # https://issues.apache.org/jira/browse/INFRA-16527
diff --git a/website/_config_test.yml b/website/_config_test.yml
deleted file mode 100644
index 551099de89b..00000000000
--- a/website/_config_test.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# Welcome to Jekyll!
-
-# This config file contains options that override the default config for testing.
-
-destination: .testcontent
diff --git a/website/build.gradle b/website/build.gradle
new file mode 100644
index 00000000000..e4581c76b4c
--- /dev/null
+++ b/website/build.gradle
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Define common lifecycle tasks and artifact types
+apply plugin: "base"
+
+task cleanWebsite(type: Delete) {
+  delete '.sass-cache', 'vendor', '.testcontent', 'content'
+}
+clean.dependsOn cleanWebsite
+
+task installBundler(type: Exec) {
+  commandLine './install_bundler.sh'
+}
+
+task installGems(type: Exec) {
+  mustRunAfter cleanWebsite, installBundler
+  inputs.file 'Gemfile.lock'
+  outputs.dir 'vendor'
+  commandLine 'bundle', 'install', '--deployment'
+}
+
+// TODO(BEAM-4696): Remove this once we can run Jenkins jobs within Docker
+task setupJenkins {
+  dependsOn installBundler, installGems
+}
+
+task buildWebsite(type: Exec) {
+  dependsOn installGems
+  mustRunAfter cleanWebsite, setupJenkins
+  inputs.files 'Gemfile.lock', '_config.yml'
+  inputs.dir 'src'
+  outputs.dir '.sass-cache'
+  outputs.dir 'content'
+  outputs.dir '.testcontent'
+  commandLine 'bundle', 'exec', 'jekyll', 'build',
+    '--config', '_config.yml',
+    '--incremental'
+}
+build.dependsOn buildWebsite
+
+task testWebsite(type: Exec) {
+  dependsOn buildWebsite
+  inputs.files 'Gemfile.lock', 'Rakefile'
+  inputs.dir '.testcontent'
+  commandLine 'bundle', 'exec', 'rake', 'test'
+}
+check.dependsOn testWebsite
+
+task preCommit {
+  dependsOn testWebsite
+}
diff --git a/website/install_bundler.sh b/website/install_bundler.sh
new file mode 100755
index 00000000000..2f4e32f7f26
--- /dev/null
+++ b/website/install_bundler.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# Install RVM.
+gpg --keyserver hkp://keys.gnupg.net --recv-keys \
+    409B6B1796C275462A1703113804BB82D39DC0E3 \
+    7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+
+curl -sSL https://get.rvm.io | bash
+source /home/jenkins/.rvm/scripts/rvm
+
+# Install Ruby.
+RUBY_VERSION_NUM=2.3.0
+rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+# Install Bundler gem
+PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+gem install bundler --user-install
+
+# Install all needed gems.
+bundle install --path ~/.gem/


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Issue Time Tracking
-------------------

    Worklog Id:     (was: 146471)
    Time Spent: 8h 50m  (was: 8h 40m)

> Create website pre-commits for apache/beam repository
> -----------------------------------------------------
>
>                 Key: BEAM-4495
>                 URL: https://issues.apache.org/jira/browse/BEAM-4495
>             Project: Beam
>          Issue Type: Sub-task
>          Components: testing, website
>            Reporter: Scott Wegner
>            Assignee: Udi Meiri
>            Priority: Major
>              Labels: beam-site-automation-reliability
>             Fix For: Not applicable
>
>          Time Spent: 8h 50m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message