drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sor...@apache.org
Subject [drill] 07/08: DRILL-7228: Upgrade to a newer version of t-digest to address inaccuracies in histogram buckets. closes #1774
Date Fri, 03 May 2019 22:46:33 GMT
This is an automated email from the ASF dual-hosted git repository.

sorabh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git

commit e191c54cab9ef969e290dc2bf95e100cb4f5b1eb
Author: Aman Sinha <asinha@maprtech.com>
AuthorDate: Tue Apr 30 12:03:53 2019 -0700

    DRILL-7228: Upgrade to a newer version of t-digest to address inaccuracies in histogram
buckets.
    closes #1774
---
 exec/java-exec/pom.xml                             |   5 +
 .../drill/exec/expr/fn/impl/TDigestFunctions.java  | 140 ++++++++++-----------
 .../impl/statistics/TDigestMergedStatistic.java    |  12 +-
 .../planner/common/NumericEquiDepthHistogram.java  |   4 +-
 4 files changed, 83 insertions(+), 78 deletions(-)

diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 2fcd935..c35e935 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -344,6 +344,11 @@
       <version>2.7.0</version>
     </dependency>
     <dependency>
+      <groupId>com.tdunning</groupId>
+      <artifactId>t-digest</artifactId>
+      <version>3.2</version>
+    </dependency>
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <exclusions>
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
index 041543b..bfcf78d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/TDigestFunctions.java
@@ -67,13 +67,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         tdigest.add(in.value);
       }
     }
@@ -81,7 +81,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -105,7 +105,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -122,13 +122,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         if (in.isSet == 1) {
           tdigest.add(in.value);
         } else {
@@ -140,7 +140,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -164,7 +164,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -181,13 +181,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         tdigest.add(in.value);
       }
     }
@@ -195,7 +195,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -219,7 +219,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -236,13 +236,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         if (in.isSet == 1) {
           tdigest.add(in.value);
         } else {
@@ -254,7 +254,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -278,7 +278,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -295,13 +295,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         tdigest.add(in.value);
       }
     }
@@ -309,7 +309,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -333,7 +333,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -350,13 +350,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         if (in.isSet == 1) {
           tdigest.add(in.value);
         } else {
@@ -368,7 +368,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -392,7 +392,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -409,13 +409,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         tdigest.add(in.value);
       }
     }
@@ -423,7 +423,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -447,7 +447,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -464,13 +464,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         if (in.isSet == 1) {
           tdigest.add(in.value);
         } else {
@@ -482,7 +482,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -506,7 +506,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -523,13 +523,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         tdigest.add(in.value);
       }
     }
@@ -537,7 +537,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -561,7 +561,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -578,13 +578,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         if (in.isSet == 1) {
           tdigest.add(in.value);
         } else {
@@ -596,7 +596,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -620,7 +620,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -637,13 +637,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         tdigest.add(in.value);
       }
     }
@@ -651,7 +651,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -675,7 +675,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -692,13 +692,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         if (in.isSet == 1) {
           tdigest.add(in.value);
         } else {
@@ -710,7 +710,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -734,7 +734,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -751,13 +751,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         tdigest.add(in.value);
       }
     }
@@ -765,7 +765,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -789,7 +789,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -806,13 +806,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         if (in.isSet == 1) {
           tdigest.add(in.value);
         } else {
@@ -824,7 +824,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -848,7 +848,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -865,13 +865,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         tdigest.add(in.value);
       }
     }
@@ -879,7 +879,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -903,7 +903,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -920,13 +920,13 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         if (in.isSet == 1) {
           tdigest.add(in.value);
         } else {
@@ -938,7 +938,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (tdigest.size() > 0) {
             int size = tdigest.smallByteSize();
@@ -962,7 +962,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 
@@ -1097,18 +1097,18 @@ public class TDigestFunctions {
     public void setup() {
       work = new ObjectHolder();
       compression.value = (int) options.getLong(org.apache.drill.exec.ExecConstants.TDIGEST_COMPRESSION);
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
 
     @Override
     public void add() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           if (in.isSet != 0) {
             byte[] buf = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(in.start,
in.end, in.buffer).getBytes();
-            com.clearspring.analytics.stream.quantile.TDigest other =
-              com.clearspring.analytics.stream.quantile.TDigest.fromBytes(java.nio.ByteBuffer.wrap(buf));
+            com.tdunning.math.stats.MergingDigest other =
+              com.tdunning.math.stats.MergingDigest.fromBytes(java.nio.ByteBuffer.wrap(buf));
             tdigest.add(other);
           }
         } catch (Exception e) {
@@ -1120,7 +1120,7 @@ public class TDigestFunctions {
     @Override
     public void output() {
       if (work.obj != null) {
-        com.clearspring.analytics.stream.quantile.TDigest tdigest = (com.clearspring.analytics.stream.quantile.TDigest)
work.obj;
+        com.tdunning.math.stats.MergingDigest tdigest = (com.tdunning.math.stats.MergingDigest)
work.obj;
         try {
           int size = tdigest.smallByteSize();
           java.nio.ByteBuffer byteBuf = java.nio.ByteBuffer.allocate(size);
@@ -1140,7 +1140,7 @@ public class TDigestFunctions {
 
     @Override
     public void reset() {
-      work.obj = new com.clearspring.analytics.stream.quantile.TDigest(compression.value);
+      work.obj = new com.tdunning.math.stats.MergingDigest(compression.value);
     }
   }
 }
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java
index cccbff6..dc84eba 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/statistics/TDigestMergedStatistic.java
@@ -20,7 +20,7 @@ package org.apache.drill.exec.physical.impl.statistics;
 // Library implementing TDigest algorithm to derive approximate quantiles. Please refer to:
 // 'Computing Extremely Accurate Quantiles using t-Digests' by Ted Dunning and Otmar Ertl
 
-import com.clearspring.analytics.stream.quantile.TDigest;
+import com.tdunning.math.stats.MergingDigest;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.server.options.OptionManager;
@@ -33,7 +33,7 @@ import java.util.Map;
 import java.nio.ByteBuffer;
 
 public class TDigestMergedStatistic extends AbstractMergedStatistic {
-  private Map<String, TDigest> tdigestHolder;
+  private Map<String, MergingDigest> tdigestHolder;
   private int compression;
 
   public TDigestMergedStatistic() {
@@ -63,7 +63,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic {
     assert (input.getField().getType().getMinorType() == TypeProtos.MinorType.MAP);
     for (ValueVector vv : input) {
       String colName = vv.getField().getName();
-      TDigest colTdigestHolder = null;
+      MergingDigest colTdigestHolder = null;
       if (tdigestHolder.get(colName) != null) {
         colTdigestHolder = tdigestHolder.get(colName);
       }
@@ -71,7 +71,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic {
       NullableVarBinaryVector.Accessor accessor = tdigestVector.getAccessor();
 
       if (!accessor.isNull(0)) {
-        TDigest other = TDigest.fromBytes(ByteBuffer.wrap(accessor.get(0)));
+        MergingDigest other = MergingDigest.fromBytes(ByteBuffer.wrap(accessor.get(0)));
         if (colTdigestHolder != null) {
           colTdigestHolder.add(other);
           tdigestHolder.put(colName, colTdigestHolder);
@@ -82,7 +82,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic {
     }
   }
 
-  public TDigest getStat(String colName) {
+  public MergingDigest getStat(String colName) {
     if (state != State.COMPLETE) {
       throw new IllegalStateException(String.format("Statistic `%s` has not completed merging
statistics",
           name));
@@ -98,7 +98,7 @@ public class TDigestMergedStatistic extends AbstractMergedStatistic {
     assert (state == State.MERGE);
     for (ValueVector outMapCol : output) {
       String colName = outMapCol.getField().getName();
-      TDigest colTdigestHolder = tdigestHolder.get(colName);
+      MergingDigest colTdigestHolder = tdigestHolder.get(colName);
       NullableVarBinaryVector vv = (NullableVarBinaryVector) outMapCol;
       vv.allocateNewSafe();
       try {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
index 9d5bf6f..0cda7c3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/NumericEquiDepthHistogram.java
@@ -26,7 +26,7 @@ import java.util.List;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexLiteral;
-import com.clearspring.analytics.stream.quantile.TDigest;
+import com.tdunning.math.stats.MergingDigest;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlOperator;
 import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
@@ -255,7 +255,7 @@ public class NumericEquiDepthHistogram implements Histogram {
   public static NumericEquiDepthHistogram buildFromTDigest(final byte[] tdigest_array,
                                                            final int numBuckets,
                                                            final long nonNullCount) {
-    TDigest tdigest = TDigest.fromBytes(java.nio.ByteBuffer.wrap(tdigest_array));
+    MergingDigest tdigest = MergingDigest.fromBytes(java.nio.ByteBuffer.wrap(tdigest_array));
 
     NumericEquiDepthHistogram histogram = new NumericEquiDepthHistogram(numBuckets);
 


Mime
View raw message