cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbel...@apache.org
Subject svn commit: r1199850 - in /cassandra/branches/cassandra-1.0: ./ src/java/org/apache/cassandra/db/ src/java/org/apache/cassandra/io/compress/ src/java/org/apache/cassandra/io/sstable/ src/java/org/apache/cassandra/tools/ test/unit/org/apache/cassandra/i...
Date Wed, 09 Nov 2011 17:03:03 GMT
Author: jbellis
Date: Wed Nov  9 17:03:03 2011
New Revision: 1199850

URL: http://svn.apache.org/viewvc?rev=1199850&view=rev
Log:
revert #3393 temporarily

Modified:
    cassandra/branches/cassandra-1.0/CHANGES.txt
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
    cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
    cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java

Modified: cassandra/branches/cassandra-1.0/CHANGES.txt
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/CHANGES.txt?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0/CHANGES.txt Wed Nov  9 17:03:03 2011
@@ -14,9 +14,9 @@ Merged from 0.8:
  * Make counter shard merging thread safe (CASSANDRA-3178)
  * Revert CASSANDRA-2855
  * Fix bug preventing the use of efficient cross-DC writes (CASSANDRA-3472)
- * Report compression ratio in CFSMBean (CASSANDRA-3393)
  * `describe ring` command for CLI (CASSANDRA-3220)
 
+
 1.0.2
  * "defragment" rows for name-based queries under STCS (CASSANDRA-2503)
  * cleanup usage of StorageService.setMode() (CASANDRA-3388)

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Wed Nov  9 17:03:03 2011
@@ -1784,12 +1784,6 @@ public class ColumnFamilyStore implement
         return data.getEstimatedColumnCountHistogram();
     }
 
-    @Override
-    public double getCompressionRatio()
-    {
-        return data.getCompressionRatio();
-    }
-
     /** true if this CFS contains secondary index data */
     public boolean isIndex()
     {

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
Wed Nov  9 17:03:03 2011
@@ -213,7 +213,6 @@ public interface ColumnFamilyStoreMBean
 
     public long[] getEstimatedRowSizeHistogram();
     public long[] getEstimatedColumnCountHistogram();
-    public double getCompressionRatio();
 
     /**
      * Returns a list of the names of the built column indexes for current store

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java Wed
Nov  9 17:03:03 2011
@@ -364,21 +364,6 @@ public class DataTracker
         return histogram;
     }
 
-    public double getCompressionRatio()
-    {
-        double sum = 0;
-        int total = 0;
-        for (SSTableReader sstable : getSSTables())
-        {
-            if (sstable.getCompressionRatio() != Double.MIN_VALUE)
-            {
-                sum += sstable.getCompressionRatio();
-                total++;
-            }
-        }
-        return total != 0 ? (double)sum/total: 0;
-    }
-
     public long getMinRowSize()
     {
         long min = 0;

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
Wed Nov  9 17:03:03 2011
@@ -23,15 +23,14 @@ import java.io.IOException;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
-import org.apache.cassandra.io.sstable.SSTableMetadata.Collector;
 import org.apache.cassandra.io.util.FileMark;
 import org.apache.cassandra.io.util.SequentialWriter;
 
 public class CompressedSequentialWriter extends SequentialWriter
 {
-    public static SequentialWriter open(String dataFilePath, String indexFilePath, boolean
skipIOCache, CompressionParameters parameters, Collector sstableMetadataCollector) throws
IOException
+    public static SequentialWriter open(String dataFilePath, String indexFilePath, boolean
skipIOCache, CompressionParameters parameters) throws IOException
     {
-        return new CompressedSequentialWriter(new File(dataFilePath), indexFilePath, skipIOCache,
parameters, sstableMetadataCollector);
+        return new CompressedSequentialWriter(new File(dataFilePath), indexFilePath, skipIOCache,
parameters);
     }
 
     // holds offset in the file where current chunk should be written
@@ -50,11 +49,7 @@ public class CompressedSequentialWriter 
 
     private final Checksum checksum = new CRC32();
 
-    private long originalSize = 0, compressedSize = 0;
-
-    private Collector sstableMetadataCollector;
-
-    public CompressedSequentialWriter(File file, String indexFilePath, boolean skipIOCache,
CompressionParameters parameters, Collector sstableMetadataCollector) throws IOException
+    public CompressedSequentialWriter(File file, String indexFilePath, boolean skipIOCache,
CompressionParameters parameters) throws IOException
     {
         super(file, parameters.chunkLength(), skipIOCache);
         this.compressor = parameters.sstableCompressor;
@@ -65,7 +60,6 @@ public class CompressedSequentialWriter 
         /* Index File (-CompressionInfo.db component) and it's header */
         metadataWriter = new CompressionMetadata.Writer(indexFilePath);
         metadataWriter.writeHeader(parameters);
-        this.sstableMetadataCollector = sstableMetadataCollector;
     }
 
     @Override
@@ -88,9 +82,6 @@ public class CompressedSequentialWriter 
         // compressing data with buffer re-use
         int compressedLength = compressor.compress(buffer, 0, validBufferBytes, compressed,
0);
 
-        originalSize += validBufferBytes;
-        compressedSize += compressedLength;
-
         // update checksum
         checksum.update(buffer, 0, validBufferBytes);
 
@@ -188,7 +179,7 @@ public class CompressedSequentialWriter 
             return; // already closed
 
         super.close();
-        sstableMetadataCollector.addCompressionRatio(compressedSize, originalSize);
+
         metadataWriter.finalizeHeader(current, chunkCount);
         metadataWriter.close();
     }

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
Wed Nov  9 17:03:03 2011
@@ -48,7 +48,7 @@ public class Descriptor
     // f (0.7.0): switched bloom filter implementations in data component
     // g (0.8): tracks flushed-at context in metadata component
     // h (1.0): tracks max client timestamp in metadata component
-    public static final String CURRENT_VERSION = "i";
+    public static final String CURRENT_VERSION = "h";
 
     public final File directory;
     public final String version;
@@ -112,11 +112,6 @@ public class Descriptor
         return version.compareTo("g") >= 0;
     }
 
-    public boolean hasCompressionRatio()
-    {
-        return version.compareTo("i") >= 0;
-    }
-
     public String filenameFor(Component component)
     {
         return filenameFor(component.name());

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
Wed Nov  9 17:03:03 2011
@@ -53,7 +53,6 @@ public class SSTableMetadata
     protected final EstimatedHistogram estimatedColumnCount;
     protected final ReplayPosition replayPosition;
     protected final long maxTimestamp;
-    protected final double compressionRatio;
     public static final SSTableMetadataSerializer serializer = new SSTableMetadataSerializer();
 
     private SSTableMetadata()
@@ -64,16 +63,15 @@ public class SSTableMetadata
     // when there is no max timestamp recorded, default to max long
     private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts,
ReplayPosition replayPosition)
     {
-        this(rowSizes, columnCounts, replayPosition, Long.MAX_VALUE, Double.MIN_VALUE);
+        this(rowSizes, columnCounts, replayPosition, Long.MAX_VALUE);
     }
 
-    private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts,
ReplayPosition replayPosition, long maxTimestamp, double cr)
+    private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram columnCounts,
ReplayPosition replayPosition, long maxTimestamp)
     {
         this.estimatedRowSize = rowSizes;
         this.estimatedColumnCount = columnCounts;
         this.replayPosition = replayPosition;
         this.maxTimestamp = maxTimestamp;
-        this.compressionRatio = cr;
     }
 
     public static SSTableMetadata createDefaultInstance()
@@ -106,11 +104,6 @@ public class SSTableMetadata
         return maxTimestamp;
     }
 
-    public double getCompressionRatio()
-    {
-        return compressionRatio;
-    }
-
     static EstimatedHistogram defaultColumnCountHistogram()
     {
         // EH of 114 can track a max value of 2395318855, i.e., > 2B columns
@@ -129,7 +122,6 @@ public class SSTableMetadata
         protected EstimatedHistogram estimatedColumnCount;
         protected ReplayPosition replayPosition;
         protected long maxTimestamp;
-        protected double compressionRatio;
 
         private Collector()
         {
@@ -137,7 +129,6 @@ public class SSTableMetadata
             this.estimatedColumnCount = defaultColumnCountHistogram();
             this.replayPosition = ReplayPosition.NONE;
             this.maxTimestamp = Long.MIN_VALUE;
-            this.compressionRatio = Double.MIN_VALUE;
         }
 
         public void addRowSize(long rowSize)
@@ -150,15 +141,6 @@ public class SSTableMetadata
             estimatedColumnCount.add(columnCount);
         }
 
-        /**
-         * Ratio is compressed/uncompressed and it is
-         * if you have 1.x then compression isn't helping
-         */
-        public void addCompressionRatio(long compressed, long uncompressed)
-        {
-            compressionRatio = (double) compressed/uncompressed;
-        }
-
         public void updateMaxTimestamp(long potentialMax)
         {
             maxTimestamp = Math.max(maxTimestamp, potentialMax);
@@ -166,7 +148,7 @@ public class SSTableMetadata
 
         public SSTableMetadata finalizeMetadata()
         {
-            return new SSTableMetadata(estimatedRowSize, estimatedColumnCount, replayPosition,
maxTimestamp, compressionRatio);
+            return new SSTableMetadata(estimatedRowSize, estimatedColumnCount, replayPosition,
maxTimestamp);
         }
 
         public Collector estimatedRowSize(EstimatedHistogram estimatedRowSize)
@@ -198,7 +180,6 @@ public class SSTableMetadata
             EstimatedHistogram.serializer.serialize(sstableStats.getEstimatedColumnCount(),
dos);
             ReplayPosition.serializer.serialize(sstableStats.getReplayPosition(), dos);
             dos.writeLong(sstableStats.getMaxTimestamp());
-            dos.writeDouble(sstableStats.getCompressionRatio());
         }
 
         public SSTableMetadata deserialize(Descriptor descriptor) throws IOException
@@ -224,14 +205,8 @@ public class SSTableMetadata
                 ReplayPosition replayPosition = descriptor.hasReplayPosition()
                                               ? ReplayPosition.serializer.deserialize(dis)
                                               : ReplayPosition.NONE;
-                long maxTimestamp = Long.MAX_VALUE;
-                double compressionRatio = Double.MIN_VALUE;
-                if (descriptor.hasCompressionRatio())
-                {
-                    maxTimestamp = dis.readLong();
-                    compressionRatio = dis.readDouble();
-                }
-                return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp,
compressionRatio);
+
+                return new SSTableMetadata(rowSizes, columnCounts, replayPosition);
             }
             finally
             {
@@ -245,7 +220,7 @@ public class SSTableMetadata
             EstimatedHistogram columnCounts = EstimatedHistogram.serializer.deserialize(dis);
             ReplayPosition replayPosition = ReplayPosition.serializer.deserialize(dis);
             long maxTimestamp = dis.readLong();
-            return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp,
Double.MIN_VALUE);
+            return new SSTableMetadata(rowSizes, columnCounts, replayPosition, maxTimestamp);
         }
 
         public long serializedSize(SSTableMetadata object)

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Wed Nov  9 17:03:03 2011
@@ -894,11 +894,6 @@ public class SSTableReader extends SSTab
         return sstableMetadata.getEstimatedColumnCount();
     }
 
-    public double getCompressionRatio()
-    {
-        return sstableMetadata.getCompressionRatio();
-    }
-
     public ReplayPosition getReplayPosition()
     {
         return sstableMetadata.getReplayPosition();

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
(original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
Wed Nov  9 17:03:03 2011
@@ -89,8 +89,7 @@ public class SSTableWriter extends SSTab
             dataFile = CompressedSequentialWriter.open(getFilename(),
                                                        descriptor.filenameFor(Component.COMPRESSION_INFO),
                                                        true,
-                                                       metadata.compressionParameters(),
-                                                       sstableMetadataCollector);
+                                                       metadata.compressionParameters());
         }
         else
         {

Modified: cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java (original)
+++ cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java Wed
Nov  9 17:03:03 2011
@@ -489,8 +489,6 @@ public class NodeCmd
                 outs.println("\t\tCompacted row minimum size: " + cfstore.getMinRowSize());
                 outs.println("\t\tCompacted row maximum size: " + cfstore.getMaxRowSize());
                 outs.println("\t\tCompacted row mean size: " + cfstore.getMeanRowSize());
-                if (cfstore.getCompressionRatio() != 0)
-                    outs.println("\t\tSSTable Compression Ratio: " + cfstore.getCompressionRatio());
 
                 outs.println("");
             }

Modified: cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
URL: http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java?rev=1199850&r1=1199849&r2=1199850&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
(original)
+++ cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
Wed Nov  9 17:03:03 2011
@@ -24,7 +24,6 @@ import java.util.concurrent.Callable;
 
 import org.junit.Test;
 
-import org.apache.cassandra.io.sstable.SSTableMetadata;
 import org.apache.cassandra.io.util.*;
 
 import static org.junit.Assert.assertEquals;
@@ -55,9 +54,8 @@ public class CompressedRandomAccessReade
 
         try
         {
-            SSTableMetadata.Collector sstableMetadataCollector = SSTableMetadata.createCollector().replayPosition(null);
             SequentialWriter writer = compressed
-                ? new CompressedSequentialWriter(f, filename + ".metadata", false, new CompressionParameters(SnappyCompressor.instance),
sstableMetadataCollector)
+                ? new CompressedSequentialWriter(f, filename + ".metadata", false, new CompressionParameters(SnappyCompressor.instance))
                 : new SequentialWriter(f, CompressionParameters.DEFAULT_CHUNK_LENGTH, false);
 
             writer.write("The quick ".getBytes());
@@ -105,8 +103,8 @@ public class CompressedRandomAccessReade
 
         File metadata = new File(file.getPath() + ".meta");
         metadata.deleteOnExit();
-        SSTableMetadata.Collector sstableMetadataCollector = SSTableMetadata.createCollector().replayPosition(null);
-        SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(),
false, new CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector);
+
+        SequentialWriter writer = new CompressedSequentialWriter(file, metadata.getPath(),
false, new CompressionParameters(SnappyCompressor.instance));
 
         writer.write(CONTENT.getBytes());
         writer.close();



Mime
View raw message