sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From linaataus...@apache.org
Subject [sentry] branch master updated: SENTRY-2491: Sentry High availability unit tests run into deadlock sometimes (Na Li, reviewed by Arjun Mishra, Kalyan Kumar Kalvagadda)
Date Mon, 28 Jan 2019 20:45:12 GMT
This is an automated email from the ASF dual-hosted git repository.

linaataustin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sentry.git


The following commit(s) were added to refs/heads/master by this push:
     new 6bfd755  SENTRY-2491: Sentry High availability unit tests run into deadlock sometimes
(Na Li, reviewed by Arjun Mishra, Kalyan Kumar Kalvagadda)
6bfd755 is described below

commit 6bfd755ced7a402eb564c15048b21f04910a144b
Author: lina.li <lina.li@cloudera.com>
AuthorDate: Mon Jan 28 14:40:18 2019 -0600

    SENTRY-2491: Sentry High availability unit tests run into deadlock sometimes (Na Li, reviewed
by Arjun Mishra, Kalyan Kumar Kalvagadda)
---
 .../tests/e2e/minisentry/InternalSentrySrv.java    | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java
b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java
index 9fa42f2..a9a514b 100644
--- a/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java
+++ b/sentry-tests/sentry-tests-hive/src/test/java/org/apache/sentry/tests/e2e/minisentry/InternalSentrySrv.java
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.sentry.api.common.ApiConstants.ClientConfig;
+import org.apache.sentry.service.common.ServiceConstants.ServerConfig;
 import org.apache.sentry.service.thrift.SentryService;
 import org.apache.sentry.service.thrift.SentryServiceFactory;
 import org.apache.thrift.protocol.TProtocol;
@@ -128,6 +129,27 @@ public class InternalSentrySrv implements SentrySrv {
     }
     for (int sentryServerNum = 0; sentryServerNum < sentryServers.size(); sentryServerNum++)
{
       start(sentryServerNum);
+
+      /*
+       * In test, we don't create schema before running the test. Instead, we use dataNucleus
to
+       * create sentry tables when they are accessed. This creates potential deadlock when
running
+       * test. For example, the following shows the event sequence that causes deadlock
+       * 1) thread_1 gets shared lock of SYSTABLES in order to read table SENTRY_HMS_NOTIFICATION_ID
+       * 2) thread_2 gets shared lock of SYSTABLES in order to read table SENTRY_HMS_NOTIFICATION_ID
+       * 3) thread_1 tries to get execution lock to create table SENTRY_HMS_NOTIFICATION_ID,
+       *    and wait for execution lock because thread_2 got shared lock already.
+       * 4) thread_2 tries to get execution lock to create table SENTRY_HMS_NOTIFICATION_ID,
+       *    and wait for execution lock because thread_1 got shared lock already.
+       * The solution is to let the instances of sentry service start with delay after the
first one.
+       * Specifically, let HMSfollower threads separates as far as possible by
+       * (the interval / sentryServers.size()).
+       *
+       * This deadlock does not exist in production because schema is created before starting
servioces.
+       * Therefore, there is no table creation after service starts.
+       */
+      if (sentryServerNum > 0) {
+        Thread.sleep(ServerConfig.SENTRY_HMSFOLLOWER_INTERVAL_MILLS_DEFAULT / sentryServers.size());
+      }
     }
   }
 


Mime
View raw message