sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ak...@apache.org
Subject [09/50] [abbrv] sentry git commit: SENTRY-1696: Expose time spent creating the initial snapshot as a metric (Alex Kolbasov, reviewed by Vamsee Yarlagadda)
Date Fri, 28 Jul 2017 17:06:27 GMT
SENTRY-1696: Expose time spent creating the initial snapshot as a metric (Alex Kolbasov, reviewed
by Vamsee Yarlagadda)


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/2fcd69aa
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/2fcd69aa
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/2fcd69aa

Branch: refs/heads/master
Commit: 2fcd69aa2ed6b0f80c5c0e91a6a84ce24459a336
Parents: 0370597
Author: Alexander Kolbasov <akolb@cloudera.com>
Authored: Wed Jul 12 19:45:52 2017 +0200
Committer: Alexander Kolbasov <akolb@cloudera.com>
Committed: Wed Jul 12 19:45:52 2017 +0200

----------------------------------------------------------------------
 .../service/thrift/FullUpdateInitializer.java   | 24 +++++++++++++++++++-
 .../sentry/service/thrift/HMSFollower.java      | 17 +++++++++++++-
 2 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/2fcd69aa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
index 1490581..b74fa50 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/FullUpdateInitializer.java
@@ -17,6 +17,7 @@
  */
 package org.apache.sentry.service.thrift;
 
+import com.codahale.metrics.Counter;
 import com.google.common.collect.ImmutableMap;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.metastore.api.Database;
@@ -25,6 +26,7 @@ import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.sentry.hdfs.PathsUpdate;
 import org.apache.sentry.hdfs.SentryMalformedPathException;
 import org.apache.sentry.hdfs.ServiceConstants.ServerConfig;
+import org.apache.sentry.provider.db.service.thrift.SentryMetrics;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,6 +48,8 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
+import static com.codahale.metrics.MetricRegistry.name;
+
 /**
  * Manage fetching full snapshot from HMS.
  * Snapshot is represented as a map from the hive object name to
@@ -103,6 +107,18 @@ public final class FullUpdateInitializer implements AutoCloseable {
           new ObjectMapping(Collections.<String, Set<String>>emptyMap());
   private final HiveConnectionFactory clientFactory;
 
+  /** Total number of database objects */
+  private final Counter databaseCount = SentryMetrics.getInstance()
+      .getCounter(name(FullUpdateInitializer.class, "total", "db"));
+
+  /** Total number of table objects */
+  private final Counter tableCount = SentryMetrics.getInstance()
+      .getCounter(name(FullUpdateInitializer.class, "total", "tables"));
+
+  /** Total number of partition objects */
+  private final Counter partitionCount = SentryMetrics.getInstance()
+      .getCounter(name(FullUpdateInitializer.class, "total", "partitions"));
+
   /**
    * Extract path (not starting with "/") from the full URI
    * @param uri - resource URI (usually with scheme)
@@ -323,7 +339,10 @@ public final class FullUpdateInitializer implements AutoCloseable {
 
           String tableName = safeIntern(tbl.getTableName().toLowerCase());
           String authzObject = (dbName + "." + tableName).intern();
-          List<String> tblPartNames = client.getClient().listPartitionNames(dbName,
tableName, (short) -1);
+          List<String> tblPartNames =
+              client.getClient().listPartitionNames(dbName, tableName, (short) -1);
+          // Count total number of partitions
+          partitionCount.inc(tblPartNames.size());
           for (int i = 0; i < tblPartNames.size(); i += maxPartitionsPerCall) {
             List<String> partsToFetch = tblPartNames.subList(i,
                     Math.min(i + maxPartitionsPerCall, tblPartNames.size()));
@@ -359,6 +378,7 @@ public final class FullUpdateInitializer implements AutoCloseable {
     DbTask(String dbName) {
       //Database names are case insensitive
       this.dbName = safeIntern(dbName.toLowerCase());
+      databaseCount.inc();
     }
 
     @Override
@@ -372,6 +392,8 @@ public final class FullUpdateInitializer implements AutoCloseable {
           return emptyObjectMapping;
         }
         List<String> allTblStr = client.getClient().getAllTables(dbName);
+        // Count total number of tables
+        tableCount.inc(allTblStr.size());
         for (int i = 0; i < allTblStr.size(); i += maxTablesPerCall) {
           List<String> tablesToFetch = allTblStr.subList(i,
                   Math.min(i + maxTablesPerCall, allTblStr.size()));

http://git-wip-us.apache.org/repos/asf/sentry/blob/2fcd69aa/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
index 2d581f7..1b6ae18 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -17,6 +17,9 @@
  */
 package org.apache.sentry.service.thrift;
 
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.Timer.Context;
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -35,6 +38,7 @@ import org.apache.sentry.hdfs.PermissionsUpdate;
 import org.apache.sentry.hdfs.service.thrift.TPrivilegeChanges;
 import org.apache.sentry.provider.db.SentryPolicyStorePlugin;
 import org.apache.sentry.provider.db.service.persistent.SentryStore;
+import org.apache.sentry.provider.db.service.thrift.SentryMetrics;
 import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
@@ -52,6 +56,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
+import static com.codahale.metrics.MetricRegistry.name;
 import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE;
 import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE;
 import static org.apache.sentry.hdfs.Updateable.Update;
@@ -77,6 +82,14 @@ public class HMSFollower implements Runnable, AutoCloseable {
   private boolean needLogHMSSupportReady = true;
   private final LeaderStatusMonitor leaderMonitor;
 
+  private static final String SNAPSHOT = "snapshot";
+  /** Measures time to get full snapshot */
+  private final Timer updateTimer = SentryMetrics.getInstance()
+      .getTimer(name(FullUpdateInitializer.class, SNAPSHOT));
+  /** Number of times update failed */
+  private final Counter failedSnapshotsCount = SentryMetrics.getInstance()
+      .getCounter(name(FullUpdateInitializer.class, "failed"));
+
   HMSFollower(Configuration conf, SentryStore store, LeaderStatusMonitor leaderMonitor,
               HiveSimpleConnectionFactory hiveConnectionFactory) {
     authzConf = conf;
@@ -299,11 +312,13 @@ public class HMSFollower implements Runnable, AutoCloseable {
     throws TException, ExecutionException {
     LOGGER.info("Request full HMS snapshot");
     try (FullUpdateInitializer updateInitializer =
-                 new FullUpdateInitializer(hiveConnectionFactory, authzConf)) {
+                 new FullUpdateInitializer(hiveConnectionFactory, authzConf);
+             Context context = updateTimer.time()) {
       Map<String, Set<String>> pathsUpdate = updateInitializer.getFullHMSSnapshot();
       LOGGER.info("Obtained full HMS snapshot");
       return pathsUpdate;
     } catch (Exception ignored) {
+      failedSnapshotsCount.inc();
       // Caller will retry later
       return Collections.emptyMap();
     }


Mime
View raw message