flink-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] asfgit closed pull request #6496: [FLINK-10063][tests] Use runit to supervise mesos processes.
Date Thu, 09 Aug 2018 08:05:38 GMT
asfgit closed pull request #6496: [FLINK-10063][tests] Use runit to supervise mesos processes.
URL: https://github.com/apache/flink/pull/6496
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/flink-jepsen/docker/Dockerfile-db b/flink-jepsen/docker/Dockerfile-db
index 1555329af3f..cb60efce2e5 100644
--- a/flink-jepsen/docker/Dockerfile-db
+++ b/flink-jepsen/docker/Dockerfile-db
@@ -21,7 +21,7 @@ FROM debian:jessie
 RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list
&& \
     apt-get update && \
     apt-get install -y -t jessie-backports openjdk-8-jdk && \
-    apt-get install -y apt-utils bzip2 curl faketime iproute iptables iputils-ping less libzip2
logrotate man man-db net-tools ntpdate psmisc python rsyslog sudo sysvinit sysvinit-core sysvinit-utils
tar unzip vim wget
+    apt-get install -y apt-utils bzip2 curl faketime iproute iptables iputils-ping less libzip2
logrotate man man-db net-tools ntpdate psmisc python rsyslog runit sudo sysvinit sysvinit-core
sysvinit-utils tar unzip vim wget
 
 RUN apt-get update && \
     apt-get -y install openssh-server && \
@@ -35,5 +35,12 @@ RUN mkdir -p /root/.ssh/ && \
     chmod 600 /root/.ssh/authorized_keys && \
     cat /root/id_rsa.pub >> /root/.ssh/authorized_keys
 
+COPY sshd-run /etc/sv/service/sshd/run
+RUN chmod +x /etc/sv/service/sshd/run && \
+    ln -sf /etc/sv/service/sshd /etc/service
+
 EXPOSE 22
-CMD exec /usr/sbin/sshd -D
+
+# Start runit process supervisor which will bring up sshd.
+# In our tests we can use runit to supervise more processes, e.g., Mesos.
+CMD runsvdir -P /etc/service /dev/null > /dev/null
diff --git a/flink-jepsen/src/jepsen/flink/db.clj b/flink-jepsen/src/jepsen/flink/db.clj
index 9a725d7149a..becc551e2cf 100644
--- a/flink-jepsen/src/jepsen/flink/db.clj
+++ b/flink-jepsen/src/jepsen/flink/db.clj
@@ -97,7 +97,7 @@
   (if (cu/exists? log-dir) (cu/ls-full log-dir) []))
 
 (defn flink-db
-  [test]
+  []
   (reify db/DB
     (setup! [_ test node]
       (c/su
@@ -131,7 +131,7 @@
   []
   (let [zk (zk/db deb-zookeeper-package)
         hadoop (hadoop/db hadoop-dist-url)
-        flink (flink-db test)]
+        flink (flink-db)]
     (combined-db [hadoop zk flink])))
 
 (defn exec-flink!
@@ -192,7 +192,7 @@
   (let [zk (zk/db deb-zookeeper-package)
         hadoop (hadoop/db hadoop-dist-url)
         mesos (mesos/db deb-mesos-package deb-marathon-package)
-        flink (flink-db test)]
+        flink (flink-db)]
     (combined-db [hadoop zk mesos flink])))
 
 (defn submit-job-with-retry!
@@ -209,24 +209,25 @@
     (let [r (fu/retry (fn []
                         (http/post
                           (str (mesos/marathon-base-url test) "/v2/apps")
-                          {:form-params  {:id   "flink"
-                                          :cmd  (str "HADOOP_CLASSPATH=`" hadoop/install-dir
"/bin/hadoop classpath` "
-                                                     "HADOOP_CONF_DIR=" hadoop/hadoop-conf-dir
" "
-                                                     install-dir "/bin/mesos-appmaster.sh
"
-                                                     "-Dmesos.master=" (zookeeper-uri
-                                                                         test
-                                                                         mesos/zk-namespace)
" "
-                                                     "-Djobmanager.rpc.address=$(hostname
-f) "
-                                                     "-Djobmanager.heap.mb=2048 "
-                                                     "-Djobmanager.rpc.port=6123 "
-                                                     "-Djobmanager.web.port=8081 "
-                                                     "-Dmesos.resourcemanager.tasks.mem=2048
"
-                                                     "-Dtaskmanager.heap.mb=2048 "
-                                                     "-Dtaskmanager.numberOfTaskSlots=2 "
-                                                     "-Dmesos.resourcemanager.tasks.cpus=1
"
-                                                     "-Drest.bind-address=$(hostname -f)
")
-                                          :cpus 1.0
-                                          :mem  2048}
+                          {:form-params  {:id                    "flink"
+                                          :cmd                   (str "HADOOP_CLASSPATH=`"
hadoop/install-dir "/bin/hadoop classpath` "
+                                                                      "HADOOP_CONF_DIR="
hadoop/hadoop-conf-dir " "
+                                                                      install-dir "/bin/mesos-appmaster.sh
"
+                                                                      "-Dmesos.master=" (zookeeper-uri
+                                                                                        
 test
+                                                                                        
 mesos/zk-namespace) " "
+                                                                      "-Djobmanager.rpc.address=$(hostname
-f) "
+                                                                      "-Djobmanager.heap.mb=2048
"
+                                                                      "-Djobmanager.rpc.port=6123
"
+                                                                      "-Djobmanager.web.port=8081
"
+                                                                      "-Dmesos.resourcemanager.tasks.mem=2048
"
+                                                                      "-Dtaskmanager.heap.mb=2048
"
+                                                                      "-Dtaskmanager.numberOfTaskSlots=2
"
+                                                                      "-Dmesos.resourcemanager.tasks.cpus=1
"
+                                                                      "-Drest.bind-address=$(hostname
-f) ")
+                                          :cpus                  1.0
+                                          :mem                   2048
+                                          :maxLaunchDelaySeconds 3}
                            :content-type :json})))]
       (info "Submitted Flink Application via Marathon" r)
       (c/on (-> test :nodes sort first)
diff --git a/flink-jepsen/src/jepsen/flink/mesos.clj b/flink-jepsen/src/jepsen/flink/mesos.clj
index fd75991bdf5..a73f25fd489 100644
--- a/flink-jepsen/src/jepsen/flink/mesos.clj
+++ b/flink-jepsen/src/jepsen/flink/mesos.clj
@@ -24,11 +24,37 @@
             [jepsen.os.debian :as debian]
             [jepsen.flink.zookeeper :refer [zookeeper-uri]]))
 
+;;; runit process supervisor (http://smarden.org/runit/)
+;;;
+;;; We use runit to supervise Mesos processes because Mesos uses a "fail-fast" approach to
+;;; error handling, e.g., the Mesos master will exit when it discovers it has been partitioned
away
+;;; from the Zookeeper quorum.
+
+(def runit-version "2.1.2-3")
+
+(defn create-supervised-service!
+  "Registers a service with the process supervisor and starts it."
+  [service-name cmd]
+  (let [service-dir (str "/etc/sv/" service-name)
+        run-script (str service-dir "/run")]
+    (c/su
+      (c/exec :mkdir :-p service-dir)
+      (c/exec :echo (clojure.string/join "\n" ["#!/bin/sh"
+                                               "exec 2>&1"
+                                               (str "exec " cmd)]) :> run-script)
+      (c/exec :chmod :+x run-script)
+      (c/exec :ln :-sf service-dir (str "/etc/service/" service-name)))))
+
+(defn stop-supervised-service!
+  "Stops a service and removes it from supervision."
+  [service-name]
+  (c/su
+    (c/exec :sv :down service-name)
+    (c/exec :rm :-f (str "/etc/service/" service-name))))
+
 ;;; Mesos
 
 (def master-count 1)
-(def master-pidfile "/var/run/mesos/master.pid")
-(def slave-pidfile "/var/run/mesos/slave.pid")
 (def master-dir "/var/lib/mesos/master")
 (def slave-dir "/var/lib/mesos/slave")
 (def log-dir "/var/log/mesos")
@@ -40,115 +66,130 @@
 
 (def marathon-bin "/usr/bin/marathon")
 (def zk-marathon-namespace "marathon")
-(def marathon-pidfile "/var/run/mesos/marathon.pid")
 (def marathon-rest-port 8080)
 
-(defn install!
-  [test node mesos-version marathon-version]
-  (c/su
-    (debian/add-repo! :mesosphere
-                      "deb http://repos.mesosphere.com/debian jessie main"
-                      "keyserver.ubuntu.com"
-                      "E56151BF")
-    (debian/install {:mesos    mesos-version
-                     :marathon marathon-version})
-    (c/exec :mkdir :-p "/var/run/mesos")
-    (c/exec :mkdir :-p master-dir)
-    (c/exec :mkdir :-p slave-dir)))
-
 ;;; Mesos functions
 
+(defn mesos-master-cmd
+  "Returns the command to run the mesos master."
+  [test node]
+  (clojure.string/join " "
+                       ["env GLOG_v=1"
+                        master-bin
+                        (str "--hostname=" (name node))
+                        (str "--log_dir=" log-dir)
+                        (str "--offer_timeout=30secs")
+                        (str "--quorum=" (util/majority master-count))
+                        (str "--registry_fetch_timeout=120secs")
+                        (str "--registry_store_timeout=5secs")
+                        (str "--work_dir=" master-dir)
+                        (str "--zk=" (zookeeper-uri test zk-namespace))]))
+
+(defn mesos-slave-cmd
+  "Returns the command to run the mesos agent."
+  [test node]
+  (clojure.string/join " "
+                       ["env GLOG_v=1"
+                        slave-bin
+                        (str "--hostname=" (name node))
+                        (str "--log_dir=" log-dir)
+                        (str "--master=" (zookeeper-uri test zk-namespace))
+                        (str "--recovery_timeout=30secs")
+                        (str "--work_dir=" slave-dir)]))
+
+(defn create-mesos-master-supervised-service!
+  [test node]
+  (create-supervised-service! "mesos-master"
+                              (mesos-master-cmd test node)))
+
+(defn create-mesos-slave-supervised-service!
+  [test node]
+  (create-supervised-service! "mesos-slave"
+                              (mesos-slave-cmd test node)))
+
+(defn master-node?
+  "Returns a truthy value if the node should run the mesos master."
+  [test node]
+  (some #{node} (take master-count (sort (:nodes test)))))
+
 (defn start-master!
   [test node]
-  (when (some #{node} (take master-count (sort (:nodes test))))
+  (when (master-node? test node)
     (info node "Starting mesos master")
     (c/su
-      (c/exec :start-stop-daemon
-              :--background
-              :--chdir master-dir
-              :--exec "/usr/bin/env"
-              :--make-pidfile
-              :--no-close
-              :--oknodo
-              :--pidfile master-pidfile
-              :--start
-              :--
-              "GLOG_v=1"
-              master-bin
-              (str "--hostname=" (name node))
-              (str "--log_dir=" log-dir)
-              (str "--offer_timeout=30secs")
-              (str "--quorum=" (util/majority master-count))
-              (str "--registry_fetch_timeout=120secs")
-              (str "--registry_store_timeout=5secs")
-              (str "--work_dir=" master-dir)
-              (str "--zk=" (zookeeper-uri test zk-namespace))
-              :>> (str log-dir "/master.stdout")
-              (c/lit "2>&1")))))
+      (create-mesos-master-supervised-service! test node))))
 
 (defn start-slave!
   [test node]
-  (when-not (some #{node} (take master-count (sort (:nodes test))))
+  (when-not (master-node? test node)
     (info node "Starting mesos slave")
     (c/su
-      (c/exec :start-stop-daemon :--start
-              :--background
-              :--chdir slave-dir
-              :--exec slave-bin
-              :--make-pidfile
-              :--no-close
-              :--pidfile slave-pidfile
-              :--oknodo
-              :--
-              (str "--hostname=" (name node))
-              (str "--log_dir=" log-dir)
-              (str "--master=" (zookeeper-uri test zk-namespace))
-              (str "--recovery_timeout=30secs")
-              (str "--work_dir=" slave-dir)
-              :>> (str log-dir "/slave.stdout")
-              (c/lit "2>&1")))))
+      (create-mesos-slave-supervised-service! test node))))
 
 (defn stop-master!
-  [node]
-  (info node "Stopping mesos master")
-  (meh (cu/grepkill! :mesos-master))
-  (meh (c/exec :rm :-rf master-pidfile))
-  (meh (c/exec :rm :-rf
-               (c/lit (str log-dir "/*"))
-               (c/lit (str master-dir "/*")))))
+  [test node]
+  (when (master-node? test node)
+    (info node "Stopping mesos master")
+    (stop-supervised-service! "mesos-master")
+    (meh (c/exec :rm :-rf
+                 (c/lit (str log-dir "/*"))
+                 (c/lit (str master-dir "/*"))))))
 
 (defn stop-slave!
-  [node]
-  (info node "Stopping mesos slave")
-  (meh (cu/grepkill! :mesos-slave))
-  (meh (c/exec :rm :-rf slave-pidfile))
-  (meh (c/exec :rm :-rf
-               (c/lit (str log-dir "/*"))
-               (c/lit (str slave-dir "/*")))))
+  [test node]
+  (when-not (master-node? test node)
+    (info node "Stopping mesos slave")
+    (stop-supervised-service! "mesos-slave")
+    (meh (c/exec :rm :-rf
+                 (c/lit (str log-dir "/*"))
+                 (c/lit (str slave-dir "/*"))))))
 
 ;;; Marathon functions
 
+(defn install!
+  [test node mesos-version marathon-version]
+  (c/su
+    (debian/add-repo! :mesosphere
+                      "deb http://repos.mesosphere.com/debian jessie main"
+                      "keyserver.ubuntu.com"
+                      "E56151BF")
+    (debian/install {:mesos    mesos-version
+                     :marathon marathon-version
+                     :runit    runit-version})
+    (c/exec :mkdir :-p "/var/run/mesos")
+    (c/exec :mkdir :-p master-dir)
+    (c/exec :mkdir :-p slave-dir)))
+
+(defn marathon-cmd
+  "Returns the command to run the marathon."
+  [test node]
+  (clojure.string/join " "
+                       [marathon-bin
+                        (str "--hostname " node)
+                        (str "--master " (zookeeper-uri test zk-namespace))
+                        (str "--zk " (zookeeper-uri test zk-marathon-namespace))
+                        (str ">> " log-dir "/marathon.out")]))
+
+(defn create-marathon-supervised-service!
+  [test node]
+  (create-supervised-service! "marathon"
+                              (marathon-cmd test node)))
+
+(defn marathon-node?
+  [test node]
+  (= node (first (sort (:nodes test)))))
+
 (defn start-marathon!
   [test node]
-  (when (= node (first (sort (:nodes test))))
+  (when (marathon-node? test node)
     (info "Start marathon")
     (c/su
-      (c/exec :start-stop-daemon :--start
-              :--background
-              :--exec marathon-bin
-              :--make-pidfile
-              :--no-close
-              :--pidfile marathon-pidfile
-              :--
-              (c/lit (str "--hostname " node))
-              (c/lit (str "--master " (zookeeper-uri test zk-namespace)))
-              (c/lit (str "--zk " (zookeeper-uri test zk-marathon-namespace)))
-              :>> (str log-dir "/marathon.stdout")
-              (c/lit "2>&1")))))
+      (create-marathon-supervised-service! test node))))
 
 (defn stop-marathon!
-  []
-  (cu/grepkill! "marathon"))
+  [test node]
+  (when (marathon-node? test node)
+    (stop-supervised-service! "marathon")))
 
 (defn marathon-base-url
   [test]
@@ -163,9 +204,9 @@
       (start-slave! test node)
       (start-marathon! test node))
     (teardown! [this test node]
-      (stop-slave! node)
-      (stop-master! node)
-      (stop-marathon!))
+      (stop-slave! test node)
+      (stop-master! test node)
+      (stop-marathon! test node))
     db/LogFiles
     (log-files [_ test node]
       (if (cu/exists? log-dir) (cu/ls-full log-dir) []))))


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message