cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dcapw...@apache.org
Subject [cassandra] branch cassandra-4.0 updated: org.apache.cassandra.db.rows.ArrayCell#unsharedHeapSizeExcludingData includes data twice
Date Tue, 31 Aug 2021 15:39:44 GMT
This is an automated email from the ASF dual-hosted git repository.

dcapwell pushed a commit to branch cassandra-4.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git


The following commit(s) were added to refs/heads/cassandra-4.0 by this push:
     new ca4f6b8  org.apache.cassandra.db.rows.ArrayCell#unsharedHeapSizeExcludingData includes
data twice
ca4f6b8 is described below

commit ca4f6b80563aeb3614fe3bce47e4fb620a8f0e8e
Author: David Capwell <dcapwell@apache.org>
AuthorDate: Mon Aug 30 16:37:53 2021 -0700

    org.apache.cassandra.db.rows.ArrayCell#unsharedHeapSizeExcludingData includes data twice
    
    patch by David Capwell; reviewed by Caleb Rackliffe for CASSANDRA-16900
---
 CHANGES.txt                                        |   1 +
 .../apache/cassandra/cql3/ColumnIdentifier.java    |   2 +-
 .../org/apache/cassandra/db/rows/ArrayCell.java    |   2 +-
 .../org/apache/cassandra/db/rows/BufferCell.java   |   6 +-
 .../org/apache/cassandra/db/rows/CellPath.java     |   2 +-
 .../org/apache/cassandra/db/rows/NativeCell.java   |   2 +-
 .../org/apache/cassandra/fql/FullQueryLogger.java  |   2 +-
 .../org/apache/cassandra/utils/ObjectSizes.java    |   8 +-
 .../unit/org/apache/cassandra/db/CellSpecTest.java | 121 +++++++++++++++++++++
 9 files changed, 138 insertions(+), 8 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index aef6fe1..f79cf30 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -2,6 +2,7 @@
  * Fix clustering order logic in CREATE MATERIALIZED VIEW (CASSANDRA-16898)
 
 4.0.1
+ * org.apache.cassandra.db.rows.ArrayCell#unsharedHeapSizeExcludingData includes data twice
(CASSANDRA-16900)
  * Exclude Jackson 1.x transitive dependency of hadoop* provided dependencies (CASSANDRA-16854)
  * Tolerate missing DNS entry when completing a host replacement (CASSANDRA-16873)
  * Harden PrunableArrayQueue against Pruner implementations that might throw exceptions (CASSANDRA-16866)
diff --git a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java
index e7bf7b9..889f23a 100644
--- a/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java
+++ b/src/java/org/apache/cassandra/cql3/ColumnIdentifier.java
@@ -205,7 +205,7 @@ public class ColumnIdentifier implements IMeasurableMemory, Comparable<ColumnIde
     public long unsharedHeapSizeExcludingData()
     {
         return EMPTY_SIZE
-             + ObjectSizes.sizeOnHeapExcludingData(bytes)
+             + ObjectSizes.sizeOfEmptyHeapByteBuffer()
              + ObjectSizes.sizeOf(text);
     }
 
diff --git a/src/java/org/apache/cassandra/db/rows/ArrayCell.java b/src/java/org/apache/cassandra/db/rows/ArrayCell.java
index edeb65c..c4fdd14 100644
--- a/src/java/org/apache/cassandra/db/rows/ArrayCell.java
+++ b/src/java/org/apache/cassandra/db/rows/ArrayCell.java
@@ -112,6 +112,6 @@ public class ArrayCell extends AbstractCell<byte[]>
 
     public long unsharedHeapSizeExcludingData()
     {
-        return EMPTY_SIZE + ObjectSizes.sizeOfArray(value) + value.length + (path == null
? 0 : path.unsharedHeapSizeExcludingData());
+        return EMPTY_SIZE + ObjectSizes.sizeOfEmptyByteArray() + (path == null ? 0 : path.unsharedHeapSizeExcludingData());
     }
 }
diff --git a/src/java/org/apache/cassandra/db/rows/BufferCell.java b/src/java/org/apache/cassandra/db/rows/BufferCell.java
index 3795b0c..55fc4b4 100644
--- a/src/java/org/apache/cassandra/db/rows/BufferCell.java
+++ b/src/java/org/apache/cassandra/db/rows/BufferCell.java
@@ -28,6 +28,8 @@ import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.ObjectSizes;
 import org.apache.cassandra.utils.memory.AbstractAllocator;
 
+import static java.lang.String.format;
+
 public class BufferCell extends AbstractCell<ByteBuffer>
 {
     private static final long EMPTY_SIZE = ObjectSizes.measure(new BufferCell(ColumnMetadata.regularColumn("",
"", "", ByteType.instance), 0L, 0, 0, ByteBufferUtil.EMPTY_BYTE_BUFFER, null));
@@ -43,7 +45,7 @@ public class BufferCell extends AbstractCell<ByteBuffer>
     {
         super(column);
         assert !column.isPrimaryKeyColumn();
-        assert column.isComplex() == (path != null);
+        assert column.isComplex() == (path != null) : format("Column %s.%s(%s: %s) isComplex:
%b with cellpath: %s", column.ksName, column.cfName, column.name, column.type.toString(),
column.isComplex(), path);
         this.timestamp = timestamp;
         this.ttl = ttl;
         this.localDeletionTime = localDeletionTime;
@@ -142,6 +144,6 @@ public class BufferCell extends AbstractCell<ByteBuffer>
 
     public long unsharedHeapSizeExcludingData()
     {
-        return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(value) + (path == null ?
0 : path.unsharedHeapSizeExcludingData());
+        return EMPTY_SIZE + ObjectSizes.sizeOfEmptyHeapByteBuffer() + (path == null ? 0 :
path.unsharedHeapSizeExcludingData());
     }
 }
diff --git a/src/java/org/apache/cassandra/db/rows/CellPath.java b/src/java/org/apache/cassandra/db/rows/CellPath.java
index 1bf8b8f..50496a1 100644
--- a/src/java/org/apache/cassandra/db/rows/CellPath.java
+++ b/src/java/org/apache/cassandra/db/rows/CellPath.java
@@ -127,7 +127,7 @@ public abstract class CellPath
 
         public long unsharedHeapSizeExcludingData()
         {
-            return EMPTY_SIZE + ObjectSizes.sizeOnHeapExcludingData(value);
+            return EMPTY_SIZE + ObjectSizes.sizeOfEmptyHeapByteBuffer();
         }
     }
 
diff --git a/src/java/org/apache/cassandra/db/rows/NativeCell.java b/src/java/org/apache/cassandra/db/rows/NativeCell.java
index 52d9ab9..02e0008 100644
--- a/src/java/org/apache/cassandra/db/rows/NativeCell.java
+++ b/src/java/org/apache/cassandra/db/rows/NativeCell.java
@@ -78,7 +78,7 @@ public class NativeCell extends AbstractCell<ByteBuffer>
         assert column.isComplex() == (path != null);
         if (path != null)
         {
-            assert path.size() == 1;
+            assert path.size() == 1 : String.format("Expected path size to be 1 but was not;
%s", path);
             size += 4 + path.get(0).remaining();
         }
 
diff --git a/src/java/org/apache/cassandra/fql/FullQueryLogger.java b/src/java/org/apache/cassandra/fql/FullQueryLogger.java
index 4b40635..9725cea 100644
--- a/src/java/org/apache/cassandra/fql/FullQueryLogger.java
+++ b/src/java/org/apache/cassandra/fql/FullQueryLogger.java
@@ -80,7 +80,7 @@ public class FullQueryLogger implements QueryEvents.Listener
     public static final String QUERIES = "queries";
     public static final String VALUES = "values";
 
-    private static final int EMPTY_BYTEBUFFER_SIZE = Ints.checkedCast(ObjectSizes.sizeOnHeapExcludingData(ByteBuffer.allocate(0)));
+    private static final int EMPTY_BYTEBUFFER_SIZE = Ints.checkedCast(ObjectSizes.sizeOfEmptyHeapByteBuffer());
 
     private static final int EMPTY_LIST_SIZE = Ints.checkedCast(ObjectSizes.measureDeep(new
ArrayList(0)));
     private static final int EMPTY_BYTEBUF_SIZE;
diff --git a/src/java/org/apache/cassandra/utils/ObjectSizes.java b/src/java/org/apache/cassandra/utils/ObjectSizes.java
index fa02ba7..04e5c65 100644
--- a/src/java/org/apache/cassandra/utils/ObjectSizes.java
+++ b/src/java/org/apache/cassandra/utils/ObjectSizes.java
@@ -37,6 +37,7 @@ public class ObjectSizes
                                              .ignoreKnownSingletons();
 
     private static final long BUFFER_EMPTY_SIZE = measure(ByteBufferUtil.EMPTY_BYTE_BUFFER);
+    private static final long BYTE_ARRAY_EMPTY_SIZE = measure(new byte[0]);
     private static final long STRING_EMPTY_SIZE = measure("");
 
     /**
@@ -128,11 +129,16 @@ public class ObjectSizes
         return BUFFER_EMPTY_SIZE + sizeOfArray(buffer.capacity(), 1);
     }
 
-    public static long sizeOnHeapExcludingData(ByteBuffer buffer)
+    public static long sizeOfEmptyHeapByteBuffer()
     {
         return BUFFER_EMPTY_SIZE;
     }
 
+    public static long sizeOfEmptyByteArray()
+    {
+        return BYTE_ARRAY_EMPTY_SIZE;
+    }
+
     /**
      * Memory a String consumes
      * @param str String to calculate memory size of
diff --git a/test/unit/org/apache/cassandra/db/CellSpecTest.java b/test/unit/org/apache/cassandra/db/CellSpecTest.java
new file mode 100644
index 0000000..44fc625
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/CellSpecTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import org.apache.cassandra.db.marshal.BytesType;
+import org.apache.cassandra.db.marshal.ListType;
+import org.apache.cassandra.db.rows.ArrayCell;
+import org.apache.cassandra.db.rows.BufferCell;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.NativeCell;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.cassandra.utils.UUIDGen;
+import org.apache.cassandra.utils.concurrent.OpOrder;
+import org.apache.cassandra.utils.memory.NativeAllocator;
+import org.apache.cassandra.utils.memory.NativePool;
+import org.assertj.core.api.Assertions;
+
+import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
+
+@RunWith(Parameterized.class)
+public class CellSpecTest
+{
+    private final Cell<?> cell;
+
+    @SuppressWarnings("unused")
+    public CellSpecTest(String ignoreOnlyUsedForBetterTestName, Cell<?> cell)
+    {
+        this.cell = cell;
+    }
+
+    @Test
+    public void unsharedHeapSizeExcludingData()
+    {
+        long empty = ObjectSizes.measure(cell);
+        long expected;
+        if (cell instanceof NativeCell)
+        {
+            // NativeCell stores the contents off-heap, so the cost on-heap is just the object's
empty case
+            expected = empty;
+        }
+        else
+        {
+            // size should be: empty + valuePtr + path.unsharedHeapSizeExcludingData() if
present
+            expected = empty + valuePtrSize(cell.value());
+            if (cell.path() != null)
+                expected += cell.path().unsharedHeapSizeExcludingData();
+        }
+
+        Assertions.assertThat(cell.unsharedHeapSizeExcludingData())
+                  .isEqualTo(expected);
+    }
+
+    private static long valuePtrSize(Object value)
+    {
+        if (value instanceof ByteBuffer)
+            return ObjectSizes.sizeOfEmptyHeapByteBuffer();
+        else if (value instanceof byte[])
+            return ObjectSizes.sizeOfEmptyByteArray();
+        throw new IllegalArgumentException("Unsupported type: " + value.getClass());
+    }
+
+    @Parameterized.Parameters(name = "{0}")
+    public static Collection<Object[]> data() {
+        TableMetadata table = TableMetadata.builder("testing", "testing")
+                                           .addPartitionKeyColumn("pk", BytesType.instance)
+                                           .build();
+
+        byte[] rawBytes = { 0, 1, 2, 3, 4, 5, 6 };
+        ByteBuffer bbBytes = ByteBuffer.wrap(rawBytes);
+        NativePool pool = new NativePool(1024, 1024, 1, () -> CompletableFuture.completedFuture(true));
+        NativeAllocator allocator = pool.newAllocator();
+        OpOrder order = new OpOrder();
+
+        List<Cell<?>> tests = new ArrayList<>();
+        BiConsumer<ColumnMetadata, CellPath> fn = (column, path) -> {
+            tests.add(new ArrayCell(column, 1234, 1, 1, rawBytes, path));
+            tests.add(new BufferCell(column, 1234, 1, 1, bbBytes, path));
+            tests.add(new NativeCell(allocator, order.getCurrent(), column, 1234, 1, 1, bbBytes,
path));
+        };
+        // simple
+        fn.accept(ColumnMetadata.regularColumn(table, bytes("simple"), BytesType.instance),
null);
+
+        // complex
+        // seems NativeCell does not allow CellPath.TOP, or CellPath.BOTTOM
+        fn.accept(ColumnMetadata.regularColumn(table, bytes("complex"), ListType.getInstance(BytesType.instance,
true)), CellPath.create(bytes(UUIDGen.getTimeUUID())));
+
+        return tests.stream().map(a -> new Object[] {a.getClass().getSimpleName() + ":"
+ (a.path() == null ? "simple" : "complex"), a}).collect(Collectors.toList());
+    }
+
+}

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message