lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mhern...@apache.org
Subject [03/50] [abbrv] git commit: resolve merge conflict
Date Tue, 24 Sep 2013 18:32:39 GMT
resolve merge conflict


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/e47e6637
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/e47e6637
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/e47e6637

Branch: refs/heads/branch_4x
Commit: e47e6637572c7b58b2a366caee7f3e7f1a308274
Parents: 64c13f3 d4b0864
Author: Mike Potts <mike@feature23.com>
Authored: Wed Jul 10 23:24:34 2013 -0400
Committer: Mike Potts <mike@feature23.com>
Committed: Wed Jul 10 23:24:34 2013 -0400

----------------------------------------------------------------------
 build/vs2012/test/Lucene.Net.Test.sln           |  58 +-
 src/contrib/Analyzers/Contrib.Analyzers.csproj  |  10 +
 src/contrib/Analyzers/Core/KeywordAnalyzer.cs   |  15 +
 src/contrib/Analyzers/Core/KeywordTokenizer.cs  |  87 +++
 .../Analyzers/Core/KeywordTokenizerFactory.cs   |  12 +
 .../Analyzers/Support/StringExtensions.cs       |  15 +
 .../Analyzers/Util/AbstractAnalysisFactory.cs   | 355 +++++++++
 src/contrib/Analyzers/Util/CharArrayMap.cs      | 480 ++++++++++++
 src/contrib/Analyzers/Util/CharArraySet.cs      |  15 +
 src/contrib/Analyzers/Util/CharacterUtils.cs    | 130 ++++
 src/contrib/Analyzers/Util/IResourceLoader.cs   |  17 +
 src/contrib/Analyzers/Util/TokenizerFactory.cs  |  11 +
 .../CompressingStoredFieldsFormat.cs            | 168 +++--
 .../CompressingStoredFieldsReader.cs            | 744 ++++++++++---------
 src/core/Codecs/Compressing/CompressionMode.cs  | 446 +++++------
 src/core/Codecs/Compressing/LZ4.cs              |   2 +-
 src/core/Index/IndexWriter.cs                   |   2 +-
 src/core/Support/Character.cs                   |  19 +
 src/core/Util/Version.cs                        |   2 +-
 test/core/Analysis/BaseTokenStreamTestCase.cs   | 257 -------
 test/core/Lucene.Net.Test.csproj                |  10 +-
 test/core/Util/LuceneTestCase.cs                | 297 --------
 .../Analysis/BaseTokenStreamTestCase.cs         | 532 +++++++++++++
 .../Lucene.Net.TestFramework.csproj             |  70 ++
 test/test-framework/Properties/AssemblyInfo.cs  |  36 +
 test/test-framework/Support/RandomizedTest.cs   |  41 +
 test/test-framework/Support/SystemProperties.cs |  26 +
 test/test-framework/Util/LuceneTestCase.cs      | 469 ++++++++++++
 28 files changed, 3100 insertions(+), 1226 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e47e6637/src/core/Codecs/Compressing/CompressingStoredFieldsReader.cs
----------------------------------------------------------------------
diff --cc src/core/Codecs/Compressing/CompressingStoredFieldsReader.cs
index 9c55e07,748efab..a4d7f7d
--- a/src/core/Codecs/Compressing/CompressingStoredFieldsReader.cs
+++ b/src/core/Codecs/Compressing/CompressingStoredFieldsReader.cs
@@@ -20,357 -20,426 +20,427 @@@ using Lucene.Net.Store
  using Lucene.Net.Support;
  using Lucene.Net.Util;
  using Lucene.Net.Util.Packed;
+ using System;
+ 
  namespace Lucene.Net.Codecs.Compressing
  {
- /**
-  * {@link StoredFieldsReader} impl for {@link CompressingStoredFieldsFormat}.
-  * @lucene.experimental
-  */
- public sealed class CompressingStoredFieldsReader: StoredFieldsReader {
- 
-   private FieldInfos fieldInfos;
-   private CompressingStoredFieldsIndexReader indexReader;
-   private IndexInput fieldsStream;
-   private int packedIntsVersion;
-   private CompressionMode compressionMode;
-   private Decompressor decompressor;
-   private BytesRef bytes;
-   private int numDocs;
-   private bool closed;
- 
-   // used by clone
-   private CompressingStoredFieldsReader(CompressingStoredFieldsReader reader) {
-     this.fieldInfos = reader.fieldInfos;
-     this.fieldsStream = (IndexInput)reader.fieldsStream.Clone();
-     this.indexReader = reader.indexReader.clone();
-     this.packedIntsVersion = reader.packedIntsVersion;
-     this.compressionMode = reader.compressionMode;
-     this.decompressor = (Decompressor)reader.decompressor.Clone();
-     this.numDocs = reader.numDocs;
-     this.bytes = new BytesRef(reader.bytes.bytes.Length);
-     this.closed = false;
-   }
- 
-   /** Sole constructor. */
-   public CompressingStoredFieldsReader(Directory d, SegmentInfo si, string segmentSuffix,
FieldInfos fn,
-       IOContext context, string formatName, CompressionMode compressionMode) 
-   {
-     this.compressionMode = compressionMode;
-     string segment = si.name;
-     bool success = false;
-     fieldInfos = fn;
-     numDocs = si.DocCount;
-     IndexInput indexStream = null;
-     try {
-       fieldsStream = d.OpenInput(IndexFileNames.SegmentFileName(segment, segmentSuffix,
FIELDS_EXTENSION), context);
-       string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION);
-       indexStream = d.OpenInput(indexStreamFN, context);
- 
-       string codecNameIdx = formatName + CODEC_SFX_IDX;
-       string codecNameDat = formatName + CODEC_SFX_DAT;
-       CodecUtil.CheckHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT);
-       CodecUtil.CheckHeader(fieldsStream, codecNameDat, VERSION_START, VERSION_CURRENT);
- 
-       indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
-       indexStream = null;
- 
-       packedIntsVersion = fieldsStream.ReadVInt();
-       decompressor = compressionMode.newDecompressor();
-       this.bytes = new BytesRef();
- 
-       success = true;
-     } finally {
-       if (!success) {
-         IOUtils.closeWhileHandlingException(this, indexStream);
-       }
-     }
-   }
- 
-   /**
-    * @throws AlreadyClosedException if this FieldsReader is closed
-    */
-   private void ensureOpen() {
-     if (closed) {
-       throw new AlreadyClosedException("this FieldsReader is closed");
-     }
-   }
- 
-   /** 
-    * Close the underlying {@link IndexInput}s.
-    */
-   public override void close() {
-     if (!closed) {
-       IOUtils.Close(fieldsStream, indexReader);
-       closed = true;
-     }
-   }
- 
-   private static void readField(ByteArrayDataInput input, StoredFieldVisitor visitor, FieldInfo
info, int bits) {
-     switch (bits & TYPE_MASK) {
-       case BYTE_ARR:
-         int length = input.readVInt();
-         byte[] data = new byte[length];
-         input.readBytes(data, 0, length);
-         visitor.binaryField(info, data);
-         break;
-       case STRING:
-         length = input.readVInt();
-         data = new byte[length];
-         input.readBytes(data, 0, length);
-         visitor.stringField(info, new string(data, IOUtils.CHARSET_UTF_8));
-         break;
-       case NUMERIC_INT:
-         visitor.intField(info, input.readInt());
-         break;
-       case NUMERIC_FLOAT:
-         visitor.floatField(info, Float.intBitsToFloat(input.readInt()));
-         break;
-       case NUMERIC_LONG:
-         visitor.longField(info, input.readLong());
-         break;
-       case NUMERIC_DOUBLE:
-         visitor.doubleField(info, Double.longBitsToDouble(input.readLong()));
-         break;
-       default:
-         throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
-     }
-   }
- 
-   private static void skipField(ByteArrayDataInput input, int bits) {
-     switch (bits & TYPE_MASK) {
-       case BYTE_ARR:
-       case STRING:
-         int length = input.readVInt();
-         input.skipBytes(length);
-         break;
-       case NUMERIC_INT:
-       case NUMERIC_FLOAT:
-         input.readInt();
-         break;
-       case NUMERIC_LONG:
-       case NUMERIC_DOUBLE:
-         input.readLong();
-         break;
-       default:
-         throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
-     }
-   }
- 
-   public override void visitDocument(int docID, StoredFieldVisitor visitor)
-   {
-     fieldsStream.Seek(indexReader.getStartPointer(docID));
- 
-     int docBase = fieldsStream.ReadVInt();
-     int chunkDocs = fieldsStream.ReadVInt();
-     if (docID < docBase
-         || docID >= docBase + chunkDocs
-         || docBase + chunkDocs > numDocs) {
-       throw new CorruptIndexException("Corrupted: docID=" + docID
-           + ", docBase=" + docBase + ", chunkDocs=" + chunkDocs
-           + ", numDocs=" + numDocs);
-     }
 +
-     int numStoredFields, length, offset, totalLength;
-     if (chunkDocs == 1) {
-       numStoredFields = fieldsStream.ReadVInt();
-       offset = 0;
-       length = fieldsStream.ReadVInt();
-       totalLength = length;
-     } else {
-       int bitsPerStoredFields = fieldsStream.ReadVInt();
-       if (bitsPerStoredFields == 0) {
-         numStoredFields = fieldsStream.ReadVInt();
-       } else if (bitsPerStoredFields > 31) {
-         throw new CorruptIndexException("bitsPerStoredFields=" + bitsPerStoredFields);
-       } else {
-         long filePointer = fieldsStream.getFilePointer();
-         PackedInts.Reader reader = PackedInts.GetDirectReaderNoHeader(fieldsStream, PackedInts.Format.PACKED,
packedIntsVersion, chunkDocs, bitsPerStoredFields);
-         numStoredFields = (int) (reader.Get(docID - docBase));
-         fieldsStream.Seek(filePointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion,
chunkDocs, bitsPerStoredFields));
-       }
- 
-       int bitsPerLength = fieldsStream.ReadVInt();
-       if (bitsPerLength == 0) {
-         length = fieldsStream.ReadVInt();
-         offset = (docID - docBase) * length;
-         totalLength = chunkDocs * length;
-       } else if (bitsPerStoredFields > 31) {
-         throw new CorruptIndexException("bitsPerLength=" + bitsPerLength);
-       } else {
-         PackedInts.ReaderIterator it = (PackedInts.ReaderIterator)PackedInts.GetReaderIteratorNoHeader(fieldsStream,
PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerLength, 1);
-         int off = 0;
-         for (int i = 0; i < docID - docBase; ++i) {
-           //TODO - HACKMP - Paul, this is a point of concern for me, in that everything
from this file, and the 
-           //decompressor.Decompress() contract is looking for int.  But, I don't want to
simply cast from long to int here.
-           off += it.Next();
+     /**
+      * {@link StoredFieldsReader} impl for {@link CompressingStoredFieldsFormat}.
+      * @lucene.experimental
+      */
+     public sealed class CompressingStoredFieldsReader : StoredFieldsReader
+     {
+ 
+         private FieldInfos fieldInfos;
+         private CompressingStoredFieldsIndexReader indexReader;
+         private IndexInput fieldsStream;
+         private int packedIntsVersion;
+         private CompressionMode compressionMode;
+         private Decompressor decompressor;
+         private BytesRef bytes;
+         private int numDocs;
+         private bool closed;
+ 
+         // used by clone
+         private CompressingStoredFieldsReader(CompressingStoredFieldsReader reader)
+         {
+             this.fieldInfos = reader.fieldInfos;
+             this.fieldsStream = (IndexInput)reader.fieldsStream.Clone();
+             this.indexReader = reader.indexReader.clone();
+             this.packedIntsVersion = reader.packedIntsVersion;
+             this.compressionMode = reader.compressionMode;
+             this.decompressor = (Decompressor)reader.decompressor.Clone();
+             this.numDocs = reader.numDocs;
+             this.bytes = new BytesRef(reader.bytes.bytes.Length);
+             this.closed = false;
          }
-         offset = off;
-         length = (int) it.Next();
-         off += length;
-         for (int i = docID - docBase + 1; i < chunkDocs; ++i) {
-           off += it.Next();
+ 
+         /** Sole constructor. */
+         public CompressingStoredFieldsReader(Directory d, SegmentInfo si, string segmentSuffix,
FieldInfos fn,
+             IOContext context, string formatName, CompressionMode compressionMode)
+         {
+             this.compressionMode = compressionMode;
+             string segment = si.name;
+             bool success = false;
+             fieldInfos = fn;
+             numDocs = si.DocCount;
+             IndexInput indexStream = null;
+             try
+             {
+                 fieldsStream = d.OpenInput(IndexFileNames.SegmentFileName(segment, segmentSuffix,
FIELDS_EXTENSION), context);
+                 string indexStreamFN = IndexFileNames.SegmentFileName(segment, segmentSuffix,
FIELDS_INDEX_EXTENSION);
+                 indexStream = d.OpenInput(indexStreamFN, context);
+ 
+                 string codecNameIdx = formatName + CODEC_SFX_IDX;
+                 string codecNameDat = formatName + CODEC_SFX_DAT;
+                 CodecUtil.CheckHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT);
+                 CodecUtil.CheckHeader(fieldsStream, codecNameDat, VERSION_START, VERSION_CURRENT);
+ 
+                 indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
+                 indexStream = null;
+ 
+                 packedIntsVersion = fieldsStream.ReadVInt();
+                 decompressor = compressionMode.newDecompressor();
+                 this.bytes = new BytesRef();
+ 
+                 success = true;
+             }
+             finally
+             {
+                 if (!success)
+                 {
+                     IOUtils.CloseWhileHandlingException((IDisposable)this, indexStream);
+                 }
+             }
          }
-         totalLength = off;
-       }
-     }
  
-     if ((length == 0) != (numStoredFields == 0)) {
-       throw new CorruptIndexException("length=" + length + ", numStoredFields=" + numStoredFields);
-     }
-     if (numStoredFields == 0) {
-       // nothing to do
-       return;
-     }
+         /**
+          * @throws AlreadyClosedException if this FieldsReader is closed
+          */
+         private void ensureOpen()
+         {
+             if (closed)
+             {
+                 throw new AlreadyClosedException("this FieldsReader is closed");
+             }
+         }
  
-     decompressor.Decompress(fieldsStream, totalLength, offset, length, bytes);
- 
-     ByteArrayDataInput documentInput = new ByteArrayDataInput(bytes.bytes, bytes.offset,
bytes.length);
-     for (int fieldIDX = 0; fieldIDX < numStoredFields; fieldIDX++) {
-       long infoAndBits = documentInput.ReadVLong();
-       int fieldNumber = Number.URShift(infoAndBits, TYPE_BITS); // (infoAndBits >>>
TYPE_BITS);
-       FieldInfo fieldInfo = fieldInfos.FieldInfo(fieldNumber);
- 
-       int bits = (int) (infoAndBits & TYPE_MASK);
- 
-       switch(visitor.NeedsField(fieldInfo)) {
-         case YES:
-           readField(documentInput, visitor, fieldInfo, bits);
-           break;
-         case NO:
-           skipField(documentInput, bits);
-           break;
-         case STOP:
-           return;
-       }
-     }
-   }
- 
-   public override StoredFieldsReader clone() {
-     ensureOpen();
-     return new CompressingStoredFieldsReader(this);
-   }
- 
-   CompressionMode getCompressionMode() {
-     return compressionMode;
-   }
- 
-   ChunkIterator chunkIterator(int startDocID) {
-     ensureOpen();
-     fieldsStream.Seek(indexReader.getStartPointer(startDocID));
-     return new ChunkIterator();
-   }
- 
-   private readonly class ChunkIterator {
- 
-     private IndexInput _indexInput;
-     private CompressingStoredFieldsReader _indexReader;
-     private int _numOfDocs;
-     private int _packedIntsVersion;
-     BytesRef bytes;
-     int docBase;
-     int chunkDocs;
-     int[] numStoredFields;
-     int[] lengths;
- 
-     public ChunkIterator(IndexInput indexInput, CompressingStoredFieldsReader indexReader,

-                             int numOfDocs, int packedIntsVersion) {
-         _indexInput = indexInput;
-         _indexReader = indexReader;
-         _numOfDocs = numOfDocs;
-         _packedIntsVersion = packedIntsVersion;
-       this.docBase = -1;
-       bytes = new BytesRef();
-       numStoredFields = new int[1];
-       lengths = new int[1];
-     }
+         /** 
+          * Close the underlying {@link IndexInput}s.
+          */
+         public override void Close()
+         {
+             if (!closed)
+             {
+                 IOUtils.Close(fieldsStream, indexReader);
+                 closed = true;
+             }
+         }
  
-     /**
-      * Return the decompressed size of the chunk
-      */
-     private int chunkSize() {
-       int sum = 0;
-       for (int i = 0; i < chunkDocs; ++i) {
-         sum += lengths[i];
-       }
-       return sum;
-     }
+         private static void ReadField(ByteArrayDataInput input, StoredFieldVisitor visitor,
FieldInfo info, int bits)
+         {
+             switch (bits & TYPE_MASK)
+             {
+                 case BYTE_ARR:
+                     int length = input.readVInt();
+                     byte[] data = new byte[length];
+                     input.readBytes(data, 0, length);
+                     visitor.binaryField(info, data);
+                     break;
+                 case STRING:
+                     length = input.readVInt();
+                     data = new byte[length];
+                     input.readBytes(data, 0, length);
+                     visitor.stringField(info, new string(data, IOUtils.CHARSET_UTF_8));
+                     break;
+                 case NUMERIC_INT:
+                     visitor.intField(info, input.readInt());
+                     break;
+                 case NUMERIC_FLOAT:
+                     visitor.floatField(info, Float.intBitsToFloat(input.readInt()));
+                     break;
+                 case NUMERIC_LONG:
+                     visitor.longField(info, input.readLong());
+                     break;
+                 case NUMERIC_DOUBLE:
+                     visitor.doubleField(info, Double.longBitsToDouble(input.readLong()));
+                     break;
+                 default:
+                     throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
+             }
+         }
  
-     /**
-      * Go to the chunk containing the provided doc ID.
-      */
-     void next(int doc) {
-       _indexInput.Seek(_indexReader.getStartPointer(doc));
- 
-       int docBase = _indexInput.ReadVInt();
-       int chunkDocs = _indexInput.ReadVInt();
-       if (docBase < this.docBase + this.chunkDocs
-           || docBase + chunkDocs > _numOfDocs) {
-         throw new CorruptIndexException("Corrupted: current docBase=" + this.docBase
-             + ", current numDocs=" + this.chunkDocs + ", new docBase=" + docBase
-             + ", new numDocs=" + chunkDocs);
-       }
-       this.docBase = docBase;
-       this.chunkDocs = chunkDocs;
- 
-       if (chunkDocs > numStoredFields.Length) {
-         int newLength = ArrayUtil.Oversize(chunkDocs, 4);
-         numStoredFields = new int[newLength];
-         lengths = new int[newLength];
-       }
- 
-       if (chunkDocs == 1) {
-           numStoredFields[0] = _indexInput.ReadVInt();
-           lengths[0] = _indexInput.ReadVInt();
-       } else {
-           int bitsPerStoredFields = _indexInput.ReadVInt();
-         if (bitsPerStoredFields == 0) {
-             Arrays.Fill(numStoredFields, 0, chunkDocs, _indexInput.ReadVInt());
-         } else if (bitsPerStoredFields > 31) {
-           throw new CorruptIndexException("bitsPerStoredFields=" + bitsPerStoredFields);
-         } else {
-             PackedInts.ReaderIterator it = (PackedInts.ReaderIterator)PackedInts.GetReaderIteratorNoHeader(_indexInput,
PackedInts.Format.PACKED, _packedIntsVersion, chunkDocs, bitsPerStoredFields, 1);
-           for (int i = 0; i < chunkDocs; ++i) {
-             numStoredFields[i] = (int) it.Next();
-           }
+         private static void SkipField(ByteArrayDataInput input, int bits)
+         {
+             switch (bits & TYPE_MASK)
+             {
+                 case BYTE_ARR:
+                 case STRING:
+                     int length = input.readVInt();
+                     input.skipBytes(length);
+                     break;
+                 case NUMERIC_INT:
+                 case NUMERIC_FLOAT:
+                     input.readInt();
+                     break;
+                 case NUMERIC_LONG:
+                 case NUMERIC_DOUBLE:
+                     input.readLong();
+                     break;
+                 default:
+                     throw new AssertionError("Unknown type flag: " + Integer.toHexString(bits));
+             }
          }
  
-         int bitsPerLength = _indexInput.ReadVInt();
-         if (bitsPerLength == 0) {
-             Arrays.Fill(lengths, 0, chunkDocs, _indexInput.ReadVInt());
-         } else if (bitsPerLength > 31) {
-           throw new CorruptIndexException("bitsPerLength=" + bitsPerLength);
-         } else {
-             PackedInts.ReaderIterator it = (PackedInts.ReaderIterator)PackedInts.GetReaderIteratorNoHeader(_indexInput,
PackedInts.Format.PACKED, _packedIntsVersion, chunkDocs, bitsPerLength, 1);
-           for (int i = 0; i < chunkDocs; ++i) {
-             lengths[i] = (int) it.Next();
-           }
+         public override void VisitDocument(int docID, StoredFieldVisitor visitor)
+         {
+             fieldsStream.Seek(indexReader.getStartPointer(docID));
+ 
+             int docBase = fieldsStream.ReadVInt();
+             int chunkDocs = fieldsStream.ReadVInt();
+             if (docID < docBase
+                 || docID >= docBase + chunkDocs
+                 || docBase + chunkDocs > numDocs)
+             {
+                 throw new CorruptIndexException("Corrupted: docID=" + docID
+                     + ", docBase=" + docBase + ", chunkDocs=" + chunkDocs
+                     + ", numDocs=" + numDocs);
+             }
+ 
+             int numStoredFields, length, offset, totalLength;
+             if (chunkDocs == 1)
+             {
+                 numStoredFields = fieldsStream.ReadVInt();
+                 offset = 0;
+                 length = fieldsStream.ReadVInt();
+                 totalLength = length;
+             }
+             else
+             {
+                 int bitsPerStoredFields = fieldsStream.ReadVInt();
+                 if (bitsPerStoredFields == 0)
+                 {
+                     numStoredFields = fieldsStream.ReadVInt();
+                 }
+                 else if (bitsPerStoredFields > 31)
+                 {
+                     throw new CorruptIndexException("bitsPerStoredFields=" + bitsPerStoredFields);
+                 }
+                 else
+                 {
+                     long filePointer = fieldsStream.FilePointer;
+                     PackedInts.Reader reader = PackedInts.GetDirectReaderNoHeader(fieldsStream,
PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerStoredFields);
+                     numStoredFields = (int)(reader.Get(docID - docBase));
+                     fieldsStream.Seek(filePointer + PackedInts.Format.PACKED.ByteCount(packedIntsVersion,
chunkDocs, bitsPerStoredFields));
+                 }
+ 
+                 int bitsPerLength = fieldsStream.ReadVInt();
+                 if (bitsPerLength == 0)
+                 {
+                     length = fieldsStream.ReadVInt();
+                     offset = (docID - docBase) * length;
+                     totalLength = chunkDocs * length;
+                 }
+                 else if (bitsPerStoredFields > 31)
+                 {
+                     throw new CorruptIndexException("bitsPerLength=" + bitsPerLength);
+                 }
+                 else
+                 {
+                     PackedInts.ReaderIterator it = (PackedInts.ReaderIterator)PackedInts.GetReaderIteratorNoHeader(fieldsStream,
PackedInts.Format.PACKED, packedIntsVersion, chunkDocs, bitsPerLength, 1);
+                     int off = 0;
+                     for (int i = 0; i < docID - docBase; ++i)
+                     {
+                         //TODO - HACKMP - Paul, this is a point of concern for me, in that
everything from this file, and the 
+                         //decompressor.Decompress() contract is looking for int.  But, I
don't want to simply cast from long to int here.
+                         off += it.Next();
+                     }
+                     offset = off;
+                     length = (int)it.Next();
+                     off += length;
+                     for (int i = docID - docBase + 1; i < chunkDocs; ++i)
+                     {
+                         off += it.Next();
+                     }
+                     totalLength = off;
+                 }
+             }
+ 
+             if ((length == 0) != (numStoredFields == 0))
+             {
+                 throw new CorruptIndexException("length=" + length + ", numStoredFields="
+ numStoredFields);
+             }
+             if (numStoredFields == 0)
+             {
+                 // nothing to do
+                 return;
+             }
+ 
+             decompressor.Decompress(fieldsStream, totalLength, offset, length, bytes);
+ 
+             ByteArrayDataInput documentInput = new ByteArrayDataInput((byte[])(Array)bytes.bytes,
bytes.offset, bytes.length);
+             for (int fieldIDX = 0; fieldIDX < numStoredFields; fieldIDX++)
+             {
+                 long infoAndBits = documentInput.ReadVLong();
+                 int fieldNumber = Number.URShift(infoAndBits, TYPE_BITS); // (infoAndBits
>>> TYPE_BITS);
+                 FieldInfo fieldInfo = fieldInfos.FieldInfo(fieldNumber);
+ 
+                 int bits = (int)(infoAndBits & TYPE_MASK);
+ 
+                 switch (visitor.NeedsField(fieldInfo))
+                 {
+                     case StoredFieldVisitor.Status.YES:
+                         ReadField(documentInput, visitor, fieldInfo, bits);
+                         break;
+                     case StoredFieldVisitor.Status.NO:
+                         SkipField(documentInput, bits);
+                         break;
+                     case StoredFieldVisitor.Status.STOP:
+                         return;
+                 }
+             }
          }
-       }
-     }
  
-     /**
-      * Decompress the chunk.
-      */
-     void decompress(){
-       // decompress data
-       int chunkSize = this.chunkSize();
-       decompressor.Decompress(_indexInput, chunkSize, 0, chunkSize, bytes);
-       if (bytes.length != chunkSize) {
-         throw new CorruptIndexException("Corrupted: expected chunk size = " + this.chunkSize()
+ ", got " + bytes.length);
-       }
-     }
+         public override StoredFieldsReader Clone()
+         {
+             ensureOpen();
+             return new CompressingStoredFieldsReader(this);
+         }
  
-     /**
-      * Copy compressed data.
-      */
-     void copyCompressedData(DataOutput output){
-       long chunkEnd = docBase + chunkDocs == numDocs
-           ? fieldsStream.length()
-           : indexReader.getStartPointer(docBase + chunkDocs);
-       output.copyBytes(fieldsStream, chunkEnd - fieldsStream.getFilePointer());
-     }
+         public CompressionMode CompressionMode
+         {
+             get
+             {
+                 return compressionMode;
+             }
+         }
  
-   }
+         // .NET Port: renamed to GetChunkIterator to avoid conflict with nested type.
+         internal ChunkIterator GetChunkIterator(int startDocID)
+         {
+             ensureOpen();
+             fieldsStream.Seek(indexReader.getStartPointer(startDocID));
+             return new ChunkIterator(fieldsStream, indexReader, numDocs, packedIntsVersion,
decompressor);
+         }
  
- }
+         internal sealed class ChunkIterator
+         {
+             private IndexInput _fieldsStream;
+             private CompressingStoredFieldsReader _indexReader;
+             private Decompressor _decompressor;
+             private int _numOfDocs;
+             private int _packedIntsVersion;
+             BytesRef bytes;
+             int docBase;
+             int chunkDocs;
+             int[] numStoredFields;
+             int[] lengths;
+ 
+             public ChunkIterator(IndexInput fieldsStream, CompressingStoredFieldsReader
indexReader,
+                                     int numOfDocs, int packedIntsVersion, Decompressor decompressor)
+             {
+                 _indexReader = indexReader;
+                 _numOfDocs = numOfDocs;
+                 _packedIntsVersion = packedIntsVersion;
+                 _decompressor = decompressor;
+                 _fieldsStream = fieldsStream;
+                 this.docBase = -1;
+                 bytes = new BytesRef();
+                 numStoredFields = new int[1];
+                 lengths = new int[1];
+             }
+ 
+             /**
+              * Return the decompressed size of the chunk
+              */
+             public int ChunkSize()
+             {
+                 int sum = 0;
+                 for (int i = 0; i < chunkDocs; ++i)
+                 {
+                     sum += lengths[i];
+                 }
+                 return sum;
+             }
+ 
+             /**
+              * Go to the chunk containing the provided doc ID.
+              */
+             public void Next(int doc)
+             {
+                 _fieldsStream.Seek(_indexReader.getStartPointer(doc));
+ 
+                 int docBase = _fieldsStream.ReadVInt();
+                 int chunkDocs = _fieldsStream.ReadVInt();
+                 if (docBase < this.docBase + this.chunkDocs
+                     || docBase + chunkDocs > _numOfDocs)
+                 {
+                     throw new CorruptIndexException("Corrupted: current docBase=" + this.docBase
+                         + ", current numDocs=" + this.chunkDocs + ", new docBase=" + docBase
+                         + ", new numDocs=" + chunkDocs);
+                 }
+                 this.docBase = docBase;
+                 this.chunkDocs = chunkDocs;
+ 
+                 if (chunkDocs > numStoredFields.Length)
+                 {
+                     int newLength = ArrayUtil.Oversize(chunkDocs, 4);
+                     numStoredFields = new int[newLength];
+                     lengths = new int[newLength];
+                 }
+ 
+                 if (chunkDocs == 1)
+                 {
+                     numStoredFields[0] = _fieldsStream.ReadVInt();
+                     lengths[0] = _fieldsStream.ReadVInt();
+                 }
+                 else
+                 {
+                     int bitsPerStoredFields = _fieldsStream.ReadVInt();
+                     if (bitsPerStoredFields == 0)
+                     {
+                         Arrays.Fill(numStoredFields, 0, chunkDocs, _fieldsStream.ReadVInt());
+                     }
+                     else if (bitsPerStoredFields > 31)
+                     {
+                         throw new CorruptIndexException("bitsPerStoredFields=" + bitsPerStoredFields);
+                     }
+                     else
+                     {
+                         PackedInts.ReaderIterator it = (PackedInts.ReaderIterator)PackedInts.GetReaderIteratorNoHeader(_fieldsStream,
PackedInts.Format.PACKED, _packedIntsVersion, chunkDocs, bitsPerStoredFields, 1);
+                         for (int i = 0; i < chunkDocs; ++i)
+                         {
+                             numStoredFields[i] = (int)it.Next();
+                         }
+                     }
+ 
+                     int bitsPerLength = _fieldsStream.ReadVInt();
+                     if (bitsPerLength == 0)
+                     {
+                         Arrays.Fill(lengths, 0, chunkDocs, _fieldsStream.ReadVInt());
+                     }
+                     else if (bitsPerLength > 31)
+                     {
+                         throw new CorruptIndexException("bitsPerLength=" + bitsPerLength);
+                     }
+                     else
+                     {
+                         PackedInts.ReaderIterator it = (PackedInts.ReaderIterator)PackedInts.GetReaderIteratorNoHeader(_fieldsStream,
PackedInts.Format.PACKED, _packedIntsVersion, chunkDocs, bitsPerLength, 1);
+                         for (int i = 0; i < chunkDocs; ++i)
+                         {
+                             lengths[i] = (int)it.Next();
+                         }
+                     }
+                 }
+             }
+ 
+             /**
+              * Decompress the chunk.
+              */
+             public void Decompress()
+             {
+                 // decompress data
+                 int chunkSize = this.ChunkSize();
+                 _decompressor.Decompress(_fieldsStream, chunkSize, 0, chunkSize, bytes);
+                 if (bytes.length != chunkSize)
+                 {
+                     throw new CorruptIndexException("Corrupted: expected chunk size = "
+ this.ChunkSize() + ", got " + bytes.length);
+                 }
+             }
+ 
+             /**
+              * Copy compressed data.
+              */
+             public void CopyCompressedData(DataOutput output)
+             {
+                 long chunkEnd = docBase + chunkDocs == _numOfDocs
+                     ? _fieldsStream.Length
+                     : _indexReader.getStartPointer(docBase + chunkDocs);
+                 output.CopyBytes(_fieldsStream, chunkEnd - _fieldsStream.FilePointer);
+             }
+ 
+         }
+ 
+     }
  }


Mime
View raw message