sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ak...@apache.org
Subject sentry git commit: SENTRY-1820: Add JSON file reporter for Sentry metrics (Alex Kolbasov, reviewed by Vamsee Yarlagadda)
Date Thu, 29 Jun 2017 03:56:43 GMT
Repository: sentry
Updated Branches:
  refs/heads/sentry-ha-redesign 4d7c4cede -> f5b4a1576


SENTRY-1820: Add JSON file reporter for Sentry metrics (Alex Kolbasov, reviewed by Vamsee
Yarlagadda)


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/f5b4a157
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/f5b4a157
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/f5b4a157

Branch: refs/heads/sentry-ha-redesign
Commit: f5b4a15763b5a181cb5fa89007b144a5a7ba4cfd
Parents: 4d7c4ce
Author: Alexander Kolbasov <akolb@cloudera.com>
Authored: Wed Jun 28 20:54:24 2017 -0700
Committer: Alexander Kolbasov <akolb@cloudera.com>
Committed: Wed Jun 28 20:54:24 2017 -0700

----------------------------------------------------------------------
 .../db/service/thrift/SentryMetrics.java        | 114 +++++++++++++++++--
 .../service/thrift/SentryServiceUtil.java       |   2 +-
 .../sentry/service/thrift/ServiceConstants.java |   5 +-
 3 files changed, 110 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/f5b4a157/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
index e3691a9..9d42d2e 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryMetrics.java
@@ -21,21 +21,37 @@ import com.codahale.metrics.*;
 
 import static com.codahale.metrics.MetricRegistry.name;
 
+import com.codahale.metrics.json.MetricsModule;
 import com.codahale.metrics.jvm.BufferPoolMetricSet;
 import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
 import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
 import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.sentry.provider.db.service.persistent.SentryStore;
 import org.apache.sentry.service.thrift.SentryService;
+import org.apache.sentry.service.thrift.SentryServiceUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static java.io.File.createTempFile;
 import static org.apache.sentry.provider.db.service.thrift.SentryMetricsServletContextListener.METRIC_REGISTRY;
 import static org.apache.sentry.service.thrift.ServiceConstants.ServerConfig;
 
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.lang.management.ManagementFactory;
 import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -43,6 +59,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
  * A singleton class which holds metrics related utility functions as well as the list of
metrics
  */
 public final class SentryMetrics {
+  public enum Reporting {
+    JMX,
+    CONSOLE,
+    LOG,
+    JSON,
+  }
+
   private static final Logger LOGGER = LoggerFactory
           .getLogger(SentryMetrics.class);
 
@@ -163,8 +186,8 @@ public final class SentryMetrics {
 
     switch(SentryMetrics.Reporting.valueOf(reporter.toUpperCase())) {
       case CONSOLE:
-        LOGGER.info(String.format("Enabled console metrics reporter with %d seconds interval",
-                reportInterval));
+        LOGGER.info("Enabled console metrics reporter with {} seconds interval",
+                reportInterval);
         final ConsoleReporter consoleReporter =
                 ConsoleReporter.forRegistry(METRIC_REGISTRY)
             .convertRatesTo(TimeUnit.SECONDS)
@@ -181,8 +204,8 @@ public final class SentryMetrics {
         jmxReporter.start();
         break;
       case LOG:
-        LOGGER.info(String.format("Enabled Log4J metrics reporter with %d seconds interval",
-                reportInterval));
+        LOGGER.info("Enabled Log4J metrics reporter with {} seconds interval",
+                reportInterval);
         final Slf4jReporter logReporter = Slf4jReporter.forRegistry(METRIC_REGISTRY)
                 .outputTo(LOGGER)
                 .convertRatesTo(TimeUnit.SECONDS)
@@ -190,8 +213,13 @@ public final class SentryMetrics {
                 .build();
         logReporter.start(reportInterval, TimeUnit.SECONDS);
         break;
+      case JSON:
+        LOGGER.info("Enabled JSON metrics reporter with {} seconds interval", reportInterval);
+        JsonFileReporter jsonReporter = new JsonFileReporter(conf, reportInterval, TimeUnit.SECONDS);
+        jsonReporter.start();
+        break;
       default:
-        LOGGER.warn("Invalid metrics reporter " + reporter);
+        LOGGER.warn("Invalid metrics reporter {}", reporter);
         break;
     }
   }
@@ -211,9 +239,77 @@ public final class SentryMetrics {
     }
   }
 
-  public enum Reporting {
-    JMX,
-    CONSOLE,
-    LOG,
+  /**
+   * Custom reporter that writes metrics as a JSON file.
+   * This class originated from Apache Hive JSON reporter.
+   */
+  private static class JsonFileReporter implements AutoCloseable, Runnable {
+    /** File permissions: -rw-r--r-- */
+    private static final short PERMISSIONS = 0644;
+    private static final String JSON_REPORTER_THREAD_NAME = "json-reporter";
+
+    private ScheduledExecutorService executor = null;
+    private final ObjectMapper jsonMapper =
+            new ObjectMapper().registerModule(new MetricsModule(TimeUnit.SECONDS,
+                    TimeUnit.MILLISECONDS,
+                    false));
+    private final Configuration conf;
+    /** Destination file name */
+    private final String pathString;
+    private final long interval;
+    private final TimeUnit unit;
+
+    JsonFileReporter(Configuration conf, long interval, TimeUnit unit) {
+      this.conf = conf;
+      pathString = conf.get(ServerConfig.SENTRY_JSON_REPORTER_FILE,
+              ServerConfig.SENTRY_JSON_REPORTER_FILE_DEFAULT);
+      this.interval = interval;
+      this.unit = unit;
+    }
+
+    private void start() {
+      executor = Executors.newScheduledThreadPool(1,
+              new ThreadFactoryBuilder().setNameFormat(JSON_REPORTER_THREAD_NAME).build());
+      executor.scheduleAtFixedRate(this, 0, interval, unit);
+    }
+
+    @Override
+    public void run() {
+      String json = null;
+      try {
+        json = jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(METRIC_REGISTRY);
+      } catch (JsonProcessingException e) {
+        LOGGER.error("Error converting metrics to JSON", e);
+        return;
+      }
+      File tmpFile = null;
+      try {
+        tmpFile = createTempFile("sentry-json", null);
+      } catch (IOException e) {
+        LOGGER.error("failed to create temp file for JSON metrics", e);
+      }
+
+      assert tmpFile != null;
+      try (LocalFileSystem fs = FileSystem.getLocal(conf);
+           BufferedWriter bw = new BufferedWriter(new FileWriter(tmpFile))) {
+        bw.write(json);
+        Path tmpPath = new Path(tmpFile.getAbsolutePath());
+        fs.setPermission(tmpPath, FsPermission.createImmutable(PERMISSIONS));
+        Path path = new Path(pathString);
+        fs.rename(tmpPath, path);
+        fs.setPermission(path, FsPermission.createImmutable(PERMISSIONS));
+      } catch (IOException e) {
+        LOGGER.warn("Error writing JSON metrics", e);
+      }
+    }
+
+    @Override
+    public void close() {
+      if (executor != null) {
+        SentryServiceUtil.shutdownAndAwaitTermination(executor,
+                JSON_REPORTER_THREAD_NAME, 1, TimeUnit.MINUTES, LOGGER);
+        executor = null;
+      }
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/f5b4a157/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
index 215f7d5..9c3e485 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/SentryServiceUtil.java
@@ -173,7 +173,7 @@ public final class SentryServiceUtil {
    * @param unit the unit of the timeout
    * @param logger the logger to log the error message if it cannot terminate. It could be
null
    */
-  static void shutdownAndAwaitTermination(ExecutorService pool, String poolName,
+  public static void shutdownAndAwaitTermination(ExecutorService pool, String poolName,
                        long timeout, TimeUnit unit, Logger logger) {
     Preconditions.checkNotNull(pool);
 

http://git-wip-us.apache.org/repos/asf/sentry/blob/f5b4a157/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
index 4f48167..83f00ca 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/ServiceConstants.java
@@ -179,7 +179,7 @@ public class ServiceConstants {
     public static final Boolean SENTRY_WEB_ENABLE_DEFAULT = false;
     public static final String SENTRY_WEB_PORT = "sentry.service.web.port";
     public static final int SENTRY_WEB_PORT_DEFAULT = 29000;
-    // Reporter is either "console" or "jmx"
+    // Reporter is either "console", "log" or "jmx"
     public static final String SENTRY_REPORTER = "sentry.service.reporter";
     public static final String SENTRY_REPORTER_JMX = SentryMetrics.Reporting.JMX.name();
//case insensitive
     public static final String SENTRY_REPORTER_CONSOLE = SentryMetrics.Reporting.CONSOLE.name();//case
insensitive
@@ -187,6 +187,9 @@ public class ServiceConstants {
     // for console reporter, reporting interval in seconds
     public static final String SENTRY_REPORTER_INTERVAL_SEC =
             "sentry.service.reporter.interval.sec";
+    public static final String SENTRY_JSON_REPORTER_FILE = "sentry.service.reporter.file";
+    public static final String SENTRY_JSON_REPORTER_FILE_DEFAULT = "/tmp/sentry-metrics.json";
+
     // Report every 5 minutes by default
     public static final int SENTRY_REPORTER_INTERVAL_DEFAULT = 300;
 


Mime
View raw message