tephra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From poo...@apache.org
Subject incubator-tephra git commit: TEPHRA-263 Enforce TTL, regardless of any in-progress transactions. Also Handle the case where TTL is longer than the duration from beginning of epoch to now.
Date Thu, 21 Sep 2017 19:00:43 GMT
Repository: incubator-tephra
Updated Branches:
  refs/heads/master 7e68ba938 -> a346efe75


TEPHRA-263 Enforce TTL, regardless of any in-progress transactions. Also Handle the case where TTL is longer than the duration from beginning of epoch to now.

This closes #61

Signed-off-by: Ali Anwar <anwar1@berkeley.edu>
Signed-off-by: poorna <poorna@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-tephra/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tephra/commit/a346efe7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tephra/tree/a346efe7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tephra/diff/a346efe7

Branch: refs/heads/master
Commit: a346efe7560cc3b96349263669c4e15951ad401b
Parents: 7e68ba9
Author: Ali Anwar <anwar1@berkeley.edu>
Authored: Fri Sep 15 17:06:07 2017 -0700
Committer: poorna <poorna@apache.org>
Committed: Thu Sep 21 12:00:23 2017 -0700

----------------------------------------------------------------------
 .../java/org/apache/tephra/util/TxUtils.java    | 30 +++++++++++++----
 .../org/apache/tephra/util/TxUtilsTest.java     | 35 ++++++++++++++++++++
 .../hbase/coprocessor/TransactionFilters.java   |  9 +++--
 .../hbase/coprocessor/TransactionProcessor.java |  2 +-
 .../TransactionVisibilityFilter.java            |  9 ++---
 .../TransactionVisibilityFilterTest.java        | 15 +++++----
 .../hbase/coprocessor/TransactionFilters.java   |  9 +++--
 .../hbase/coprocessor/TransactionProcessor.java |  2 +-
 .../TransactionVisibilityFilter.java            |  7 ++--
 .../TransactionVisibilityFilterTest.java        | 15 +++++----
 .../hbase/coprocessor/TransactionFilters.java   |  9 +++--
 .../hbase/coprocessor/TransactionProcessor.java |  2 +-
 .../TransactionVisibilityFilter.java            |  7 ++--
 .../TransactionVisibilityFilterTest.java        | 15 +++++----
 .../hbase/coprocessor/TransactionFilters.java   |  9 +++--
 .../hbase/coprocessor/TransactionProcessor.java |  2 +-
 .../TransactionVisibilityFilter.java            |  7 ++--
 .../TransactionVisibilityFilterTest.java        | 15 +++++----
 .../hbase/coprocessor/TransactionFilters.java   |  9 +++--
 .../hbase/coprocessor/TransactionProcessor.java |  2 +-
 .../TransactionVisibilityFilter.java            | 11 +++---
 .../TransactionVisibilityFilterTest.java        | 15 +++++----
 .../hbase/coprocessor/TransactionFilters.java   |  9 +++--
 .../hbase/coprocessor/TransactionProcessor.java |  2 +-
 .../TransactionVisibilityFilter.java            | 11 +++---
 .../TransactionVisibilityFilterTest.java        | 15 +++++----
 26 files changed, 177 insertions(+), 96 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java b/tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java
index aaca23d..5eb2b3e 100644
--- a/tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java
+++ b/tephra-core/src/main/java/org/apache/tephra/util/TxUtils.java
@@ -19,6 +19,8 @@
 package org.apache.tephra.util;
 
 import com.google.common.primitives.Longs;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.tephra.Transaction;
 import org.apache.tephra.TransactionManager;
 import org.apache.tephra.TransactionType;
@@ -61,9 +63,7 @@ public class TxUtils {
    * @return The oldest timestamp that will be visible for the given transaction and TTL configuration
    */
   public static long getOldestVisibleTimestamp(Map<byte[], Long> ttlByFamily, Transaction tx) {
-    long maxTTL = getMaxTTL(ttlByFamily);
-    // we know that data will not be cleaned up while this tx is running up to this point as janitor uses it
-    return maxTTL < Long.MAX_VALUE ? tx.getVisibilityUpperBound() - maxTTL * TxConstants.MAX_TX_PER_MS : 0;
+    return getOldestVisibleTimestamp(ttlByFamily, tx, false);
   }
 
   /**
@@ -75,12 +75,28 @@ public class TxUtils {
    * @return The oldest timestamp that will be visible for the given transaction and TTL configuration
    */
   public static long getOldestVisibleTimestamp(Map<byte[], Long> ttlByFamily, Transaction tx, boolean readNonTxnData) {
-    if (readNonTxnData) {
-      long maxTTL = getMaxTTL(ttlByFamily);
-      return maxTTL < Long.MAX_VALUE ? System.currentTimeMillis() - maxTTL : 0;
+    long maxTTL = getMaxTTL(ttlByFamily);
+    if (maxTTL == Long.MAX_VALUE) {
+      return 0;
     }
+    return getOldestVisibleTimestamp(maxTTL, tx, readNonTxnData);
+  }
 
-    return getOldestVisibleTimestamp(ttlByFamily, tx);
+  /**
+   * Returns the oldest visible timestamp for the given transaction, based on the TTL configured.  If the TTL is
+   * negative or zero, the oldest visible timestamp will be {@code 0}.
+   * @param ttl TTL value (in milliseconds)
+   * @param tx The current transaction
+   * @param readNonTxnData indicates that the timestamp returned should allow reading non-transactional data
+   * @return The oldest timestamp that will be visible for the given transaction and TTL configuration
+   */
+  public static long getOldestVisibleTimestamp(long ttl, Transaction tx, boolean readNonTxnData) {
+    if (ttl <= 0) {
+      return 0;
+    }
+    long ttlFactor = readNonTxnData ? 1 : TxConstants.MAX_TX_PER_MS;
+    // if the computed ttl is negative, return 0 because timestamps can not be negative
+    return Math.max(0, tx.getTransactionId() - ttl * ttlFactor);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-core/src/test/java/org/apache/tephra/util/TxUtilsTest.java
----------------------------------------------------------------------
diff --git a/tephra-core/src/test/java/org/apache/tephra/util/TxUtilsTest.java b/tephra-core/src/test/java/org/apache/tephra/util/TxUtilsTest.java
index db687fe..b172e89 100644
--- a/tephra-core/src/test/java/org/apache/tephra/util/TxUtilsTest.java
+++ b/tephra-core/src/test/java/org/apache/tephra/util/TxUtilsTest.java
@@ -19,9 +19,14 @@
 package org.apache.tephra.util;
 
 import org.apache.tephra.Transaction;
+import org.apache.tephra.TxConstants;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
 import static org.junit.Assert.assertEquals;
 
 /**
@@ -54,4 +59,34 @@ public class TxUtilsTest {
     tx = new Transaction(100, 110, new long[] {}, new long[] {}, Transaction.NO_TX_IN_PROGRESS);
     Assert.assertEquals(99, TxUtils.getPruneUpperBound(tx));
   }
+
+  @Test
+  public void testTTL() {
+    long txIdsPerSecond = 1000 * TxConstants.MAX_TX_PER_MS;
+    byte[] family = new byte[] { 'd' };
+    long currentTxTimeSeconds = 100;
+    Transaction tx = new Transaction(100 * txIdsPerSecond, currentTxTimeSeconds * txIdsPerSecond,
+                                     new long[] {10 * txIdsPerSecond, 30 * txIdsPerSecond},
+                                     new long[] {80 * txIdsPerSecond, 90 * txIdsPerSecond},
+                                     80 * txIdsPerSecond);
+    int ttlSeconds = 60;
+    Map<byte[], Long> ttlByFamily = Collections.singletonMap(family, ttlSeconds * 1000L);
+    // ttl should only be impacted by the current transaction's id, and not by any older, in-progress transactions
+    Assert.assertEquals((currentTxTimeSeconds - ttlSeconds) * txIdsPerSecond,
+                        TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx));
+  }
+
+  @Test
+  public void testLargeTTL() {
+    long txIdsPerSecond = 1000 * TxConstants.MAX_TX_PER_MS;
+    byte[] family = new byte[] { 'd' };
+    long currentTxTimeSeconds = 100;
+    Transaction tx = new Transaction(100 * txIdsPerSecond, currentTxTimeSeconds * txIdsPerSecond,
+                                     new long[] { }, new long[] { }, 100);
+    // ~100 years, so that the computed start timestamp is prior to 0 (epoch)
+    long ttlSeconds = TimeUnit.DAYS.toSeconds(365 * 100);
+    Map<byte[], Long> ttlByFamily = Collections.singletonMap(family, ttlSeconds * 1000L);
+    // oldest visible timestamp should be 0, not negative, because HBase timestamps can not be negative
+    Assert.assertEquals(0, TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx));
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
index 0ca9f9c..c46082f 100644
--- a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
@@ -40,7 +40,8 @@ public class TransactionFilters {
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                            ScanType scanType) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, false,
+                                                              scanType, null));
   }
 
   /**
@@ -50,13 +51,15 @@ public class TransactionFilters {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                                           ScanType scanType, @Nullable Filter cellFilter) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+                                           boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData,
+                                                              scanType, cellFilter));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
index 10ecfa4..d78ccea 100644
--- a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
@@ -446,7 +446,7 @@ public class TransactionProcessor extends BaseRegionObserver {
    * @param type the type of scan operation being performed
    */
   protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
-    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData, type, filter);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
index 3675268..d24f504 100644
--- a/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
+++ b/tephra-hbase-compat-0.96/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
@@ -70,7 +70,7 @@ public class TransactionVisibilityFilter extends FilterBase {
    */
   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                      ScanType scanType) {
-    this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+    this(tx, ttlByFamily, allowEmptyValues, false, scanType, null);
   }
 
   /**
@@ -80,19 +80,20 @@ public class TransactionVisibilityFilter extends FilterBase {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
-   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                               ScanType scanType, @Nullable Filter cellFilter) {
+  public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+                                     boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
     this.tx = tx;
     this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
     for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
       long familyTTL = ttlEntry.getValue();
       oldestTsByFamily.put(ttlEntry.getKey(),
-                           familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+                           TxUtils.getOldestVisibleTimestamp(familyTTL, tx, readNonTxnData));
     }
     this.allowEmptyValues = allowEmptyValues;
     this.clearDeletes =

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
index 4d34ed9..7286a2a 100644
--- a/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
+++ b/tephra-hbase-compat-0.96/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
@@ -74,7 +74,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     TxFilterFactory txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -94,7 +94,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, skipFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -114,7 +114,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeNextFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -134,7 +134,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, nextColFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -294,8 +294,9 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     ttls.put(FAM2, 30L);
     ttls.put(FAM3, 0L);
 
-    Transaction tx = txManager.startShort();
-    long now = tx.getVisibilityUpperBound();
+    long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+    // we explicitly set the readPointer to 'now', because if you set it to an older value, it can filter values out
+    Transaction tx = new Transaction(now, now, new long[0], new long[0], now);
     Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
     assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
                  filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
@@ -353,7 +354,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
   private class CustomTxFilter extends TransactionVisibilityFilter {
     public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
                           @Nullable Filter cellFilter) {
-      super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
+      super(tx, ttlByFamily, allowEmptyValues, false, scanType, cellFilter);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
index 0ca9f9c..c46082f 100644
--- a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
@@ -40,7 +40,8 @@ public class TransactionFilters {
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                            ScanType scanType) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, false,
+                                                              scanType, null));
   }
 
   /**
@@ -50,13 +51,15 @@ public class TransactionFilters {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                                           ScanType scanType, @Nullable Filter cellFilter) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+                                           boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData,
+                                                              scanType, cellFilter));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
index 17d55a4..435ae02 100644
--- a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
@@ -446,7 +446,7 @@ public class TransactionProcessor extends BaseRegionObserver {
    * @param type the type of scan being performed
    */
   protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
-    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData, type, filter);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
index 9a617a9..3e3db59 100644
--- a/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
+++ b/tephra-hbase-compat-0.98/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
@@ -70,7 +70,7 @@ public class TransactionVisibilityFilter extends FilterBase {
    */
   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                      ScanType scanType) {
-    this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+    this(tx, ttlByFamily, allowEmptyValues, false, scanType, null);
   }
 
   /**
@@ -80,19 +80,20 @@ public class TransactionVisibilityFilter extends FilterBase {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                              ScanType scanType, @Nullable Filter cellFilter) {
+                                     boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
     this.tx = tx;
     this.oldestTsByFamily = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
     for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
       long familyTTL = ttlEntry.getValue();
       oldestTsByFamily.put(ttlEntry.getKey(),
-                           familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+                           TxUtils.getOldestVisibleTimestamp(familyTTL, tx, readNonTxnData));
     }
     this.allowEmptyValues = allowEmptyValues;
     this.clearDeletes =

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
index 3352eef..49de0eb 100644
--- a/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
+++ b/tephra-hbase-compat-0.98/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
@@ -74,7 +74,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     TxFilterFactory txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -94,7 +94,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, skipFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -114,7 +114,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeNextFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -134,7 +134,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, nextColFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -293,8 +293,9 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     ttls.put(FAM2, 30L);
     ttls.put(FAM3, 0L);
 
-    Transaction tx = txManager.startShort();
-    long now = tx.getVisibilityUpperBound();
+    long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+    // we explicitly set the readPointer to 'now', because if you set it to an older value, it can filter values out
+    Transaction tx = new Transaction(now, now, new long[0], new long[0], now);
     Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
     assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
                  filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
@@ -352,7 +353,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
   private class CustomTxFilter extends TransactionVisibilityFilter {
     public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
                           @Nullable Filter cellFilter) {
-      super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
+      super(tx, ttlByFamily, allowEmptyValues, false, scanType, cellFilter);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
index 0ca9f9c..c46082f 100644
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
@@ -40,7 +40,8 @@ public class TransactionFilters {
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                            ScanType scanType) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, false,
+                                                              scanType, null));
   }
 
   /**
@@ -50,13 +51,15 @@ public class TransactionFilters {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                                           ScanType scanType, @Nullable Filter cellFilter) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+                                           boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData,
+                                                              scanType, cellFilter));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
index ca96052..ab2ac8d 100644
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
@@ -446,7 +446,7 @@ public class TransactionProcessor extends BaseRegionObserver {
    * @param type the type of scan being performed
    */
   protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
-    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData, type, filter);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
index 9825ded..9e32478 100644
--- a/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
+++ b/tephra-hbase-compat-1.0-cdh/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
@@ -73,7 +73,7 @@ public class TransactionVisibilityFilter extends FilterBase {
    */
   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                      ScanType scanType) {
-    this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+    this(tx, ttlByFamily, allowEmptyValues, false, scanType, null);
   }
 
   /**
@@ -83,19 +83,20 @@ public class TransactionVisibilityFilter extends FilterBase {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                              ScanType scanType, @Nullable Filter cellFilter) {
+                                     boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
     this.tx = tx;
     this.oldestTsByFamily = Maps.newTreeMap();
     for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
       long familyTTL = ttlEntry.getValue();
       oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()),
-                           familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+                           TxUtils.getOldestVisibleTimestamp(familyTTL, tx, readNonTxnData));
     }
     this.allowEmptyValues = allowEmptyValues;
     this.clearDeletes =

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
index c27a10d..d4e4ed1 100644
--- a/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
+++ b/tephra-hbase-compat-1.0-cdh/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
@@ -74,7 +74,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     TxFilterFactory txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -94,7 +94,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, skipFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -114,7 +114,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeNextFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -134,7 +134,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, nextColFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -293,8 +293,9 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     ttls.put(FAM2, 30L);
     ttls.put(FAM3, 0L);
 
-    Transaction tx = txManager.startShort();
-    long now = tx.getVisibilityUpperBound();
+    long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+    // we explicitly set the readPointer to 'now', because if you set it to an older value, it can filter values out
+    Transaction tx = new Transaction(now, now, new long[0], new long[0], now);
     Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
     assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
                  filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
@@ -352,7 +353,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
   private class CustomTxFilter extends TransactionVisibilityFilter {
     public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
                           @Nullable Filter cellFilter) {
-      super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
+      super(tx, ttlByFamily, allowEmptyValues, false, scanType, cellFilter);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
index 0ca9f9c..c46082f 100644
--- a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
@@ -40,7 +40,8 @@ public class TransactionFilters {
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                            ScanType scanType) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, false,
+                                                              scanType, null));
   }
 
   /**
@@ -50,13 +51,15 @@ public class TransactionFilters {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                                           ScanType scanType, @Nullable Filter cellFilter) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+                                           boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData,
+                                                              scanType, cellFilter));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
index 263ee98..7325a7a 100644
--- a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
@@ -446,7 +446,7 @@ public class TransactionProcessor extends BaseRegionObserver {
    * @param type the type of scan being performed
    */
   protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
-    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData, type, filter);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
index 9825ded..9e32478 100644
--- a/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
+++ b/tephra-hbase-compat-1.0/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
@@ -73,7 +73,7 @@ public class TransactionVisibilityFilter extends FilterBase {
    */
   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                      ScanType scanType) {
-    this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+    this(tx, ttlByFamily, allowEmptyValues, false, scanType, null);
   }
 
   /**
@@ -83,19 +83,20 @@ public class TransactionVisibilityFilter extends FilterBase {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                              ScanType scanType, @Nullable Filter cellFilter) {
+                                     boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
     this.tx = tx;
     this.oldestTsByFamily = Maps.newTreeMap();
     for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
       long familyTTL = ttlEntry.getValue();
       oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()),
-                           familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+                           TxUtils.getOldestVisibleTimestamp(familyTTL, tx, readNonTxnData));
     }
     this.allowEmptyValues = allowEmptyValues;
     this.clearDeletes =

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
index 4b2b40c..28dfba8 100644
--- a/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
+++ b/tephra-hbase-compat-1.0/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
@@ -74,7 +74,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     TxFilterFactory txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -94,7 +94,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, skipFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -114,7 +114,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeNextFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -134,7 +134,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, nextColFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -293,8 +293,9 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     ttls.put(FAM2, 30L);
     ttls.put(FAM3, 0L);
 
-    Transaction tx = txManager.startShort();
-    long now = tx.getVisibilityUpperBound();
+    long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+    // we explicitly set the readPointer to 'now', because if you set it to an older value, it can filter values out
+    Transaction tx = new Transaction(now, now, new long[0], new long[0], now);
     Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
     assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
                  filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
@@ -352,7 +353,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
   private class CustomTxFilter extends TransactionVisibilityFilter {
     public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
                           @Nullable Filter cellFilter) {
-      super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
+      super(tx, ttlByFamily, allowEmptyValues, false, scanType, cellFilter);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java b/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
index 0ca9f9c..c46082f 100644
--- a/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
+++ b/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
@@ -40,7 +40,8 @@ public class TransactionFilters {
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                            ScanType scanType) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, false,
+                                                              scanType, null));
   }
 
   /**
@@ -50,13 +51,15 @@ public class TransactionFilters {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                                           ScanType scanType, @Nullable Filter cellFilter) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+                                           boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData,
+                                                              scanType, cellFilter));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
index 553f598..848cb1f 100644
--- a/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
@@ -446,7 +446,7 @@ public class TransactionProcessor extends BaseRegionObserver {
    * @param type the type of scan being performed
    */
   protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
-    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData, type, filter);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
index 5ad7c29..bd3c719 100644
--- a/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
+++ b/tephra-hbase-compat-1.1-base/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
@@ -70,8 +70,8 @@ public class TransactionVisibilityFilter extends FilterBase {
    * @param scanType the type of scan operation being performed
    */
   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                              ScanType scanType) {
-    this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+                                     ScanType scanType) {
+    this(tx, ttlByFamily, allowEmptyValues, false, scanType, null);
   }
 
   /**
@@ -81,19 +81,20 @@ public class TransactionVisibilityFilter extends FilterBase {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
-   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                               ScanType scanType, @Nullable Filter cellFilter) {
+  public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+                                     boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
     this.tx = tx;
     this.oldestTsByFamily = Maps.newTreeMap();
     for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
       long familyTTL = ttlEntry.getValue();
       oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()),
-                           familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+                           TxUtils.getOldestVisibleTimestamp(familyTTL, tx, readNonTxnData));
     }
     this.allowEmptyValues = allowEmptyValues;
     this.clearDeletes =

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.1-base/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.1-base/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-1.1-base/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
index 1b02609..b6650f6 100644
--- a/tephra-hbase-compat-1.1-base/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
+++ b/tephra-hbase-compat-1.1-base/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
@@ -74,7 +74,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     TxFilterFactory txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -94,7 +94,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, skipFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -114,7 +114,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeNextFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -134,7 +134,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, nextColFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -294,8 +294,9 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     ttls.put(FAM2, 30L);
     ttls.put(FAM3, 0L);
 
-    Transaction tx = txManager.startShort();
-    long now = tx.getVisibilityUpperBound();
+    long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+    // we explicitly set the readPointer to 'now', because if you set it to an older value, it can filter values out
+    Transaction tx = new Transaction(now, now, new long[0], new long[0], now);
     Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
     assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
                  filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
@@ -353,7 +354,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
   private class CustomTxFilter extends TransactionVisibilityFilter {
     public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
                           @Nullable Filter cellFilter) {
-      super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
+      super(tx, ttlByFamily, allowEmptyValues, false, scanType, cellFilter);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java b/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
index 0ca9f9c..c46082f 100644
--- a/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
+++ b/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionFilters.java
@@ -40,7 +40,8 @@ public class TransactionFilters {
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
                                            ScanType scanType) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, null));
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, false,
+                                                              scanType, null));
   }
 
   /**
@@ -50,13 +51,15 @@ public class TransactionFilters {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
   public static Filter getVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                                           ScanType scanType, @Nullable Filter cellFilter) {
-    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter));
+                                           boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
+    return new CellSkipFilter(new TransactionVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData,
+                                                              scanType, cellFilter));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java b/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
index 553f598..848cb1f 100644
--- a/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
+++ b/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionProcessor.java
@@ -446,7 +446,7 @@ public class TransactionProcessor extends BaseRegionObserver {
    * @param type the type of scan being performed
    */
   protected Filter getTransactionFilter(Transaction tx, ScanType type, Filter filter) {
-    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, type, filter);
+    return TransactionFilters.getVisibilityFilter(tx, ttlByFamily, allowEmptyValues, readNonTxnData, type, filter);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java b/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
index 40e2c37..81eb604 100644
--- a/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
+++ b/tephra-hbase-compat-1.3/src/main/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilter.java
@@ -70,8 +70,8 @@ public class TransactionVisibilityFilter extends FilterBase {
    * @param scanType the type of scan operation being performed
    */
   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                              ScanType scanType) {
-    this(tx, ttlByFamily, allowEmptyValues, scanType, null);
+                                     ScanType scanType) {
+    this(tx, ttlByFamily, allowEmptyValues, false, scanType, null);
   }
 
   /**
@@ -81,19 +81,20 @@ public class TransactionVisibilityFilter extends FilterBase {
    * @param ttlByFamily map of time-to-live (TTL) (in milliseconds) by column family name
    * @param allowEmptyValues if {@code true} cells with empty {@code byte[]} values will be returned, if {@code false}
    *                         these will be interpreted as "delete" markers and the column will be filtered out
+   * @param readNonTxnData whether data written before Tephra was enabled on a table should be readable
    * @param scanType the type of scan operation being performed
    * @param cellFilter if non-null, this filter will be applied to all cells visible to the current transaction, by
    *                   calling {@link Filter#filterKeyValue(org.apache.hadoop.hbase.Cell)}.  If null, then
    *                   {@link Filter.ReturnCode#INCLUDE_AND_NEXT_COL} will be returned instead.
    */
-   public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
-                               ScanType scanType, @Nullable Filter cellFilter) {
+  public TransactionVisibilityFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues,
+                                     boolean readNonTxnData, ScanType scanType, @Nullable Filter cellFilter) {
     this.tx = tx;
     this.oldestTsByFamily = Maps.newTreeMap();
     for (Map.Entry<byte[], Long> ttlEntry : ttlByFamily.entrySet()) {
       long familyTTL = ttlEntry.getValue();
       oldestTsByFamily.put(new ImmutableBytesWritable(ttlEntry.getKey()),
-                           familyTTL <= 0 ? 0 : tx.getVisibilityUpperBound() - familyTTL * TxConstants.MAX_TX_PER_MS);
+                           TxUtils.getOldestVisibleTimestamp(familyTTL, tx, readNonTxnData));
     }
     this.allowEmptyValues = allowEmptyValues;
     this.clearDeletes =

http://git-wip-us.apache.org/repos/asf/incubator-tephra/blob/a346efe7/tephra-hbase-compat-1.3/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
----------------------------------------------------------------------
diff --git a/tephra-hbase-compat-1.3/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java b/tephra-hbase-compat-1.3/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
index 1b02609..b6650f6 100644
--- a/tephra-hbase-compat-1.3/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
+++ b/tephra-hbase-compat-1.3/src/test/java/org/apache/tephra/hbase/coprocessor/TransactionVisibilityFilterTest.java
@@ -74,7 +74,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     TxFilterFactory txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -94,7 +94,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, skipFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, skipFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -114,7 +114,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, includeNextFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, includeNextFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -134,7 +134,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     txFilterFactory = new TxFilterFactory() {
       @Override
       public Filter getTxFilter(Transaction tx, Map<byte[], Long> familyTTLs) {
-        return new TransactionVisibilityFilter(tx, familyTTLs, false, ScanType.USER_SCAN, nextColFilter);
+        return new TransactionVisibilityFilter(tx, familyTTLs, false, false, ScanType.USER_SCAN, nextColFilter);
       }
     };
     runFilteringTest(txFilterFactory,
@@ -294,8 +294,9 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
     ttls.put(FAM2, 30L);
     ttls.put(FAM3, 0L);
 
-    Transaction tx = txManager.startShort();
-    long now = tx.getVisibilityUpperBound();
+    long now = System.currentTimeMillis() * TxConstants.MAX_TX_PER_MS;
+    // we explicitly set the readPointer to 'now', because if you set it to an older value, it can filter values out
+    Transaction tx = new Transaction(now, now, new long[0], new long[0], now);
     Filter filter = new TransactionVisibilityFilter(tx, ttls, false, ScanType.USER_SCAN);
     assertEquals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL,
                  filter.filterKeyValue(newKeyValue("row1", FAM, "val1", now)));
@@ -353,7 +354,7 @@ public class TransactionVisibilityFilterTest extends AbstractTransactionVisibili
   private class CustomTxFilter extends TransactionVisibilityFilter {
     public CustomTxFilter(Transaction tx, Map<byte[], Long> ttlByFamily, boolean allowEmptyValues, ScanType scanType,
                           @Nullable Filter cellFilter) {
-      super(tx, ttlByFamily, allowEmptyValues, scanType, cellFilter);
+      super(tx, ttlByFamily, allowEmptyValues, false, scanType, cellFilter);
     }
 
     @Override


Mime
View raw message