cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r1004303 - in /cassandra/branches/cassandra-0.6: CHANGES.txt src/java/org/apache/cassandra/io/IndexSummary.java src/java/org/apache/cassandra/io/SSTableReader.java
Date Mon, 04 Oct 2010 16:10:11 GMT
Author: jbellis
Date: Mon Oct  4 16:10:11 2010
New Revision: 1004303

URL: http://svn.apache.org/viewvc?rev=1004303&view=rev
Log:
optimize SSTableReader.loadIndexFile to reduce re-reading of index data during startup.  patch
by jbellis and Stu Hood for CASSANDRA-1526

Modified:
    cassandra/branches/cassandra-0.6/CHANGES.txt
    cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
    cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java

Modified: cassandra/branches/cassandra-0.6/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/CHANGES.txt?rev=1004303&r1=1004302&r2=1004303&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.6/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.6/CHANGES.txt Mon Oct  4 16:10:11 2010
@@ -39,6 +39,8 @@
  * allow nodes to change IPs between restarts (CASSANDRA-1518)
  * remove assertion causing rare (and harmless) error messages in
    commitlog (CASSANDRA-1330)
+ * optimize SSTableReader.loadIndexFile to reduce re-reading of index
+   data during server startup (CASSANDRA-1526)
 
 
 0.6.5

Modified: cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java?rev=1004303&r1=1004302&r2=1004303&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java (original)
+++ cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/IndexSummary.java Mon
Oct  4 16:10:11 2010
@@ -31,7 +31,7 @@ import org.apache.cassandra.db.Decorated
 
 public class IndexSummary
 {
-    private ArrayList<KeyPosition> indexPositions;
+    private ArrayList<KeyPosition> indexPositions = new ArrayList<KeyPosition>();
     private Map<KeyPosition, SSTable.PositionSize> spannedIndexDataPositions;
     private Map<Long, KeyPosition> spannedIndexPositions;
     private int keysWritten = 0;
@@ -43,10 +43,6 @@ public class IndexSummary
                                     && SSTableReader.bufferIndex(indexPosition) !=
SSTableReader.bufferIndex(nextIndexPosition);
         if ((keysWritten++ % DatabaseDescriptor.getIndexInterval() == 0) || spannedIndexEntry)
         {
-            if (indexPositions == null)
-            {
-                indexPositions  = new ArrayList<KeyPosition>();
-            }
             KeyPosition info = new KeyPosition(decoratedKey, indexPosition);
             indexPositions.add(info);
 

Modified: cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java?rev=1004303&r1=1004302&r2=1004303&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java (original)
+++ cassandra/branches/cassandra-0.6/src/java/org/apache/cassandra/io/SSTableReader.java Mon
Oct  4 16:10:11 2010
@@ -19,26 +19,29 @@
 package org.apache.cassandra.io;
 
 import java.io.*;
-import java.util.*;
-import java.lang.ref.ReferenceQueue;
 import java.lang.ref.Reference;
-import java.nio.channels.FileChannel;
+import java.lang.ref.ReferenceQueue;
 import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 
 import org.apache.log4j.Logger;
 
 import org.apache.cassandra.cache.InstrumentedCache;
-import org.apache.cassandra.dht.IPartitioner;
-import org.apache.cassandra.utils.BloomFilter;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.io.util.BufferedRandomAccessFile;
 import org.apache.cassandra.io.util.FileDataInput;
 import org.apache.cassandra.io.util.MappedFileDataInput;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Pair;
 
 /**
  * SSTableReaders are open()ed by Table.onStart; after that they are created by SSTableWriter.renameAndOpen.
@@ -244,30 +247,36 @@ public class SSTableReader extends SSTab
         try
         {
             long indexSize = input.length();
+            // we need to know both the current index entry and its data position, as well
as the
+            // next such pair, in order to compute tne mmap-spanning entries.  since seeking
+            // backwards in a 0.6 BRAF is expensive, we make one pass through by reading
the "next"
+            // entry in each loop through, then summarizing the previous one.
+            IndexSummary.KeyPosition thisEntry = null, nextEntry = null;
+            long thisDataPos = -1, nextDataPos = -1;
             while (true)
             {
                 long indexPosition = input.getFilePointer();
                 if (indexPosition == indexSize)
-                {
                     break;
-                }
-                DecoratedKey decoratedKey = partitioner.convertFromDiskFormat(input.readUTF());
+
+                DecoratedKey key = partitioner.convertFromDiskFormat(input.readUTF());
                 long dataPosition = input.readLong();
-                long nextIndexPosition = input.getFilePointer();
-                // read the next index entry to see how big the row is
-                long nextDataPosition;
-                if (input.isEOF())
+                if (thisEntry == null)
                 {
-                    nextDataPosition = length();
-                }
-                else
-                {
-                    input.readUTF();
-                    nextDataPosition = input.readLong();
-                    input.seek(nextIndexPosition);
+                    thisEntry = new IndexSummary.KeyPosition(key, indexPosition);
+                    thisDataPos = dataPosition;
+                    continue;
                 }
-                indexSummary.maybeAddEntry(decoratedKey, dataPosition, nextDataPosition -
dataPosition, indexPosition, nextIndexPosition);
+
+                nextEntry = new IndexSummary.KeyPosition(key, indexPosition);
+                nextDataPos = dataPosition;
+                indexSummary.maybeAddEntry(thisEntry.key, thisDataPos, nextDataPos - thisDataPos,
thisEntry.indexPosition, nextEntry.indexPosition);
+
+                thisEntry = nextEntry;
+                thisDataPos = nextDataPos;
             }
+            assert thisEntry != null; // should not have any zero-row sstables
+            indexSummary.maybeAddEntry(thisEntry.key, thisDataPos, length() - thisDataPos,
thisEntry.indexPosition, input.length());
             indexSummary.complete();
         }
         finally



Mime
View raw message