cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bdeggles...@apache.org
Subject [5/6] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.11
Date Tue, 06 Nov 2018 19:25:30 GMT
Merge branch 'cassandra-3.0' into cassandra-3.11


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/a6a9dce1
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/a6a9dce1
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/a6a9dce1

Branch: refs/heads/cassandra-3.11
Commit: a6a9dce157a4ed14e7d08e854c504423dd199daa
Parents: 69f8cc7 d60c783
Author: Blake Eggleston <bdeggleston@gmail.com>
Authored: Tue Nov 6 11:17:47 2018 -0800
Committer: Blake Eggleston <bdeggleston@gmail.com>
Committed: Tue Nov 6 11:19:04 2018 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 src/java/org/apache/cassandra/db/Slice.java     |  25 +-----
 .../cassandra/io/sstable/format/Version.java    |   2 +
 .../io/sstable/format/big/BigFormat.java        |   9 ++
 .../io/sstable/metadata/MetadataCollector.java  |  25 +++---
 .../io/sstable/metadata/StatsMetadata.java      |  14 ++-
 .../mc-1-big-CompressionInfo.db                 | Bin 0 -> 43 bytes
 .../mc-1-big-Data.db                            | Bin 0 -> 65 bytes
 .../mc-1-big-Digest.crc32                       |   1 +
 .../mc-1-big-Filter.db                          | Bin 0 -> 16 bytes
 .../mc-1-big-Index.db                           | Bin 0 -> 8 bytes
 .../mc-1-big-Statistics.db                      | Bin 0 -> 4789 bytes
 .../mc-1-big-Summary.db                         | Bin 0 -> 56 bytes
 .../mc-1-big-TOC.txt                            |   8 ++
 .../db/SinglePartitionSliceCommandTest.java     |  87 +++++++++++++++++++
 .../cassandra/io/sstable/LegacySSTableTest.java |  34 +++++++-
 16 files changed, 166 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 03abb5b,0fb1b86..f923fa0
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,5 -1,5 +1,6 @@@
 -3.0.18
 +3.11.4
 +Merged from 3.0:
+  * Sstable min/max metadata can cause data loss (CASSANDRA-14861)
   * Dropped columns can cause reverse sstable iteration to return prematurely (CASSANDRA-14838)
   * Legacy sstables with  multi block range tombstones create invalid bound sequences (CASSANDRA-14823)
   * Expand range tombstone validation checks to multiple interim request stages (CASSANDRA-14824)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/db/Slice.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index e61f4b3,f48d0a6..a618c96
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@@ -19,8 -19,8 +19,9 @@@ package org.apache.cassandra.io.sstable
  
  import java.nio.ByteBuffer;
  import java.util.ArrayList;
+ import java.util.Arrays;
  import java.util.Collections;
 +import java.util.EnumMap;
  import java.util.List;
  import java.util.Map;
  
@@@ -93,8 -95,8 +97,8 @@@ public class MetadataCollector implemen
      protected double compressionRatio = NO_COMPRESSION_RATIO;
      protected StreamingHistogram.StreamingHistogramBuilder estimatedTombstoneDropTime =
defaultTombstoneDropTimeHistogramBuilder();
      protected int sstableLevel;
-     protected ByteBuffer[] minClusteringValues;
-     protected ByteBuffer[] maxClusteringValues;
 -    private ClusteringPrefix minClustering = Slice.Bound.TOP;
 -    private ClusteringPrefix maxClustering = Slice.Bound.BOTTOM;
++    private ClusteringPrefix minClustering = ClusteringBound.TOP;
++    private ClusteringPrefix maxClustering = ClusteringBound.BOTTOM;
      protected boolean hasLegacyCounterShards = false;
      protected long totalColumnsSet;
      protected long totalRows;
@@@ -277,7 -271,8 +273,8 @@@
  
      public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner,
double bloomFilterFPChance, long repairedAt, SerializationHeader header)
      {
+         Preconditions.checkState(comparator.compare(maxClustering, minClustering) >=
0);
 -        Map<MetadataType, MetadataComponent> components = Maps.newHashMap();
 +        Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class);
          components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
          components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
                                                               estimatedCellPerPartitionCount,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
index 7ad6198,2891687..1bdbcb2
--- a/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
+++ b/test/unit/org/apache/cassandra/db/SinglePartitionSliceCommandTest.java
@@@ -37,11 -43,12 +43,13 @@@ import org.apache.cassandra.SchemaLoade
  import org.apache.cassandra.Util;
  import org.apache.cassandra.config.CFMetaData;
  import org.apache.cassandra.config.ColumnDefinition;
 +import org.apache.cassandra.config.DatabaseDescriptor;
  import org.apache.cassandra.config.Schema;
  import org.apache.cassandra.cql3.ColumnIdentifier;
+ import org.apache.cassandra.cql3.QueryOptions;
  import org.apache.cassandra.cql3.QueryProcessor;
  import org.apache.cassandra.cql3.UntypedResultSet;
+ import org.apache.cassandra.cql3.statements.SelectStatement;
  import org.apache.cassandra.db.filter.AbstractClusteringIndexFilter;
  import org.apache.cassandra.db.filter.ClusteringIndexNamesFilter;
  import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter;
@@@ -387,4 -388,81 +397,81 @@@ public class SinglePartitionSliceComman
          Assert.assertNotNull(ret);
          Assert.assertFalse(ret.isEmpty());
      }
+ 
+ 
+     public static List<Unfiltered> getUnfilteredsFromSinglePartition(String q)
+     {
+         SelectStatement stmt = (SelectStatement) QueryProcessor.parseStatement(q).prepare(ClientState.forInternalCalls()).statement;
+ 
+         List<Unfiltered> unfiltereds = new ArrayList<>();
+         SinglePartitionReadCommand.Group query = (SinglePartitionReadCommand.Group) stmt.getQuery(QueryOptions.DEFAULT,
FBUtilities.nowInSeconds());
+         Assert.assertEquals(1, query.commands.size());
+         SinglePartitionReadCommand command = Iterables.getOnlyElement(query.commands);
 -        try (ReadOrderGroup group = ReadOrderGroup.forCommand(command);
 -             UnfilteredPartitionIterator partitions = command.executeLocally(group))
++        try (ReadExecutionController controller = ReadExecutionController.forCommand(command);
++             UnfilteredPartitionIterator partitions = command.executeLocally(controller))
+         {
+             assert partitions.hasNext();
+             try (UnfilteredRowIterator partition = partitions.next())
+             {
+                 while (partition.hasNext())
+                 {
+                     Unfiltered next = partition.next();
+                     unfiltereds.add(next);
+                 }
+             }
+             assert !partitions.hasNext();
+         }
+         return unfiltereds;
+     }
+ 
+     private static void assertQueryReturnsSingleRT(String query)
+     {
+         List<Unfiltered> unfiltereds = getUnfilteredsFromSinglePartition(query);
+         Assert.assertEquals(2, unfiltereds.size());
+         Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker());
+         Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false));
+         Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker());
+         Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false));
+     }
+ 
+     private static ByteBuffer bb(int v)
+     {
+         return Int32Type.instance.decompose(v);
+     }
+ 
+     /**
+      * tests the bug raised in CASSANDRA-14861, where the sstable min/max can
+      * exclude range tombstones for clustering ranges not also covered by rows
+      */
+     @Test
+     public void sstableFiltering()
+     {
+         QueryProcessor.executeOnceInternal("CREATE TABLE ks.legacy_mc_inaccurate_min_max
(k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))");
+         CFMetaData metadata = Schema.instance.getCFMetaData("ks", "legacy_mc_inaccurate_min_max");
+         ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(metadata.cfId);
+ 
+         QueryProcessor.executeOnceInternal("INSERT INTO ks.legacy_mc_inaccurate_min_max
(k, c1, c2, c3, v) VALUES (100, 2, 2, 2, 2)");
+         QueryProcessor.executeOnceInternal("DELETE FROM ks.legacy_mc_inaccurate_min_max
WHERE k=100 AND c1=1");
+         assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE
k=100 AND c1=1 AND c2=1");
+         cfs.forceBlockingFlush();
+         assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE
k=100 AND c1=1 AND c2=1");
+ 
+         assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE
k=100 AND c1=1 AND c2=1 AND c3=1"); // clustering names
+ 
+         cfs.truncateBlocking();
+ 
+         long nowMillis = System.currentTimeMillis();
 -        Slice slice = Slice.make(new Clustering(bb(2), bb(3)), new Clustering(bb(10), bb(10)));
++        Slice slice = Slice.make(Clustering.make(bb(2), bb(3)), Clustering.make(bb(10),
bb(10)));
+         RangeTombstone rt = new RangeTombstone(slice, new DeletionTime(TimeUnit.MILLISECONDS.toMicros(nowMillis),
+                                                                        Ints.checkedCast(TimeUnit.MILLISECONDS.toSeconds(nowMillis))));
+         PartitionUpdate update = new PartitionUpdate(cfs.metadata, bb(100), cfs.metadata.partitionColumns(),
1);
+         update.add(rt);
+         new Mutation(update).apply();
+ 
+         assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE
k=100 AND c1=3 AND c2=2");
+         cfs.forceBlockingFlush();
+         assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE
k=100 AND c1=3 AND c2=2");
+         assertQueryReturnsSingleRT("SELECT * FROM ks.legacy_mc_inaccurate_min_max WHERE
k=100 AND c1=3 AND c2=2 AND c3=2"); // clustering names
+ 
+     }
  }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/a6a9dce1/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
index dae7823,5c65b31..bd51c0f
--- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java
@@@ -35,12 -36,20 +36,21 @@@ import org.slf4j.Logger
  import org.slf4j.LoggerFactory;
  
  import org.apache.cassandra.SchemaLoader;
 +import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.cql3.QueryOptions;
  import org.apache.cassandra.cql3.QueryProcessor;
  import org.apache.cassandra.cql3.UntypedResultSet;
+ import org.apache.cassandra.cql3.statements.SelectStatement;
  import org.apache.cassandra.db.ColumnFamilyStore;
  import org.apache.cassandra.db.Keyspace;
 -import org.apache.cassandra.db.ReadOrderGroup;
++import org.apache.cassandra.db.ReadExecutionController;
+ import org.apache.cassandra.db.SinglePartitionReadCommand;
+ import org.apache.cassandra.db.SinglePartitionSliceCommandTest;
  import org.apache.cassandra.db.compaction.Verifier;
+ import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
+ import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+ import org.apache.cassandra.db.rows.Unfiltered;
+ import org.apache.cassandra.db.rows.UnfilteredRowIterator;
  import org.apache.cassandra.dht.IPartitioner;
  import org.apache.cassandra.dht.Range;
  import org.apache.cassandra.dht.Token;
@@@ -278,7 -268,29 +289,28 @@@ public class LegacySSTableTes
          }
      }
  
 -
      @Test
+     public void testInaccurateSSTableMinMax() throws Exception
+     {
+         QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_mc_inaccurate_min_max
(k int, c1 int, c2 int, c3 int, v int, primary key (k, c1, c2, c3))");
+         loadLegacyTable("legacy_%s_inaccurate_min_max%s", "mc", "");
+ 
+         /*
+          sstable has the following mutations:
+             INSERT INTO legacy_tables.legacy_mc_inaccurate_min_max (k, c1, c2, c3, v) VALUES
(100, 4, 4, 4, 4)
+             DELETE FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100 AND c1<3
+          */
+ 
+         String query = "SELECT * FROM legacy_tables.legacy_mc_inaccurate_min_max WHERE k=100
AND c1=1 AND c2=1";
+         List<Unfiltered> unfiltereds = SinglePartitionSliceCommandTest.getUnfilteredsFromSinglePartition(query);
+         Assert.assertEquals(2, unfiltereds.size());
+         Assert.assertTrue(unfiltereds.get(0).isRangeTombstoneMarker());
+         Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(0)).isOpen(false));
+         Assert.assertTrue(unfiltereds.get(1).isRangeTombstoneMarker());
+         Assert.assertTrue(((RangeTombstoneMarker) unfiltereds.get(1)).isClose(false));
+     }
+ 
+     @Test
      public void testVerifyOldSSTables() throws Exception
      {
          for (String legacyVersion : legacyVersions)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message