cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r803135 - in /incubator/cassandra/trunk: conf/ src/java/org/apache/cassandra/config/ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/db/filter/ src/java/org/apache/cassandra/io/
Date Tue, 11 Aug 2009 14:13:47 GMT
Author: jbellis
Date: Tue Aug 11 14:13:47 2009
New Revision: 803135

URL: http://svn.apache.org/viewvc?rev=803135&view=rev
Log:
add settings for buffer sizes; make unconfigurable ones more sane.
patch by jbellis; reviewed by Jun Rao and Eric Evans for CASSANDRA-355

Modified:
    incubator/cassandra/trunk/conf/storage-conf.xml
    incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
    incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java

Modified: incubator/cassandra/trunk/conf/storage-conf.xml
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/conf/storage-conf.xml?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/conf/storage-conf.xml (original)
+++ incubator/cassandra/trunk/conf/storage-conf.xml Tue Aug 11 14:13:47 2009
@@ -198,6 +198,12 @@
     <!-- Memory, Disk, and Performance                                        -->
     <!--======================================================================-->
 
+    <!-- Buffer size to use when performing contiguous column slices. 
+         Increase this to the size of the column slices you typically 
+         perform. (Name-based queries are performed with a buffer size
+         of ColumnIndexSizeInKB.) -->
+    <SlicedBufferSizeInKB>64</SlicedBufferSizeInKB>
+
     <!-- Buffer size to use when flushing memtables to disk.
          (Only one memtable is ever flushed at a time.)
          Increase (decrease) the index buffer size relative to the data buffer

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
Tue Aug 11 14:13:47 2009
@@ -71,6 +71,7 @@
 
     private static double flushDataBufferSizeInMB_ = 32;
     private static double flushIndexBufferSizeInMB_ = 8;
+    private static int slicedReadBufferSizeInKB_ = 64;
     private static List<String> tables_ = new ArrayList<String>();
     private static Set<String> applicationColumnFamilies_ = new HashSet<String>();
 
@@ -239,6 +240,12 @@
                 flushIndexBufferSizeInMB_ = Double.parseDouble(rawFlushIndex);
             }
 
+            String rawSlicedBuffer = xmlUtils.getNodeValue("/Storage/SlicedBufferSizeInKB");
+            if (rawSlicedBuffer != null)
+            {
+                slicedReadBufferSizeInKB_ = Integer.parseInt(rawSlicedBuffer);
+            }
+
             /* TCP port on which the storage system listens */
             String port = xmlUtils.getNodeValue("/Storage/StoragePort");
             if ( port != null )
@@ -956,4 +963,14 @@
     {
         return flushIndexBufferSizeInMB_;
     }
+
+    public static int getIndexedReadBufferSizeInKB()
+    {
+        return columnIndexSizeInKB_;
+    }
+
+    public static int getSlicedReadBufferSizeInKB()
+    {
+        return slicedReadBufferSizeInKB_;
+    }
 }

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java Tue Aug 11 14:13:47
2009
@@ -249,7 +249,8 @@
 
         for (File file : clogs)
         {
-            BufferedRandomAccessFile reader = new BufferedRandomAccessFile(file.getAbsolutePath(),
"r");
+            int bufferSize = (int)Math.min(file.length(), 32 * 1024 * 1024);
+            BufferedRandomAccessFile reader = new BufferedRandomAccessFile(file.getAbsolutePath(),
"r", bufferSize);
             CommitLogHeader clHeader = readCommitLogHeader(reader);
             /* seek to the lowest position */
             int lowPos = CommitLogHeader.getLowestPosition(clHeader);

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableNamesIterator.java
Tue Aug 11 14:13:47 2009
@@ -27,7 +27,7 @@
         if (position < 0)
             return;
 
-        BufferedRandomAccessFile file = new BufferedRandomAccessFile(filename, "r");
+        BufferedRandomAccessFile file = new BufferedRandomAccessFile(filename, "r", DatabaseDescriptor.getIndexedReadBufferSizeInKB()
* 1024);
         try
         {
             file.seek(position);

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/filter/SSTableSliceIterator.java
Tue Aug 11 14:13:47 2009
@@ -7,6 +7,7 @@
 import org.apache.cassandra.db.ColumnFamily;
 import org.apache.cassandra.db.marshal.AbstractType;
 import org.apache.cassandra.io.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
 import com.google.common.collect.AbstractIterator;
 
 /**
@@ -86,7 +87,7 @@
 
         public ColumnGroupReader(String filename, String key, long position) throws IOException
         {
-            this.file = new BufferedRandomAccessFile(filename, "r");
+            this.file = new BufferedRandomAccessFile(filename, "r", DatabaseDescriptor.getSlicedReadBufferSizeInKB()
* 1024);
 
             file.seek(position);
             String keyInDisk = file.readUTF();

Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java
URL: http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java?rev=803135&r1=803134&r2=803135&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java (original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/io/FileStruct.java Tue Aug 11
14:13:47 2009
@@ -19,10 +19,12 @@
 package org.apache.cassandra.io;
 
 import java.io.IOException;
+import java.io.File;
 import java.util.Iterator;
 
 import org.apache.cassandra.db.IColumn;
 import org.apache.cassandra.db.ColumnFamily;
+import org.apache.cassandra.config.DatabaseDescriptor;
 
 import org.apache.log4j.Logger;
 import com.google.common.collect.AbstractIterator;
@@ -40,7 +42,11 @@
 
     FileStruct(SSTableReader sstable) throws IOException
     {
-        this.file = new BufferedRandomAccessFile(sstable.getFilename(), "r", 1024 * 1024);
+        // TODO this is used for both compactions and key ranges.  the buffer sizes we want
+        // to use for these ops are very different.  here we are leaning towards the key-range
+        // use case since that is more common.  What we really want is to split those
+        // two uses of this class up.
+        this.file = new BufferedRandomAccessFile(sstable.getFilename(), "r", 256 * 1024);
         this.sstable = sstable;
     }
 



Mime
View raw message