lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [21/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests: Removed \core directory and put its contents in root directory
Date Sun, 26 Feb 2017 23:37:09 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestPayloads.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestPayloads.cs b/src/Lucene.Net.Tests/Index/TestPayloads.cs
new file mode 100644
index 0000000..5c106d9
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestPayloads.cs
@@ -0,0 +1,738 @@
+using Lucene.Net.Analysis.TokenAttributes;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Text;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+
+    using Lucene.Net.Analysis;
+    using Lucene.Net.Support;
+    using Lucene.Net.Util;
+    using NUnit.Framework;
+    using System.IO;
+    using IBits = Lucene.Net.Util.IBits;
+    using BytesRef = Lucene.Net.Util.BytesRef;
+    using Directory = Lucene.Net.Store.Directory;
+    using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator;
+    using Document = Documents.Document;
+    using Field = Field;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+    using PayloadAttribute = Lucene.Net.Analysis.TokenAttributes.PayloadAttribute;
+    using TestUtil = Lucene.Net.Util.TestUtil;
+    using TextField = TextField;
+
+    [TestFixture]
+    public class TestPayloads : LuceneTestCase
+    {
+        // Simple tests to test the payloads
+        [Test]
+        public virtual void TestPayload()
+        {
+            BytesRef payload = new BytesRef("this is a test!");
+            Assert.AreEqual(payload.Length, "this is a test!".Length, "Wrong payload length.");
+
+            BytesRef clone = (BytesRef)payload.Clone();
+            Assert.AreEqual(payload.Length, clone.Length);
+            for (int i = 0; i < payload.Length; i++)
+            {
+                Assert.AreEqual(payload.Bytes[i + payload.Offset], clone.Bytes[i + clone.Offset]);
+            }
+        }
+
+        // Tests whether the DocumentWriter and SegmentMerger correctly enable the
+        // payload bit in the FieldInfo
+        [Test]
+        public virtual void TestPayloadFieldBit()
+        {
+            Directory ram = NewDirectory();
+            PayloadAnalyzer analyzer = new PayloadAnalyzer();
+            IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
+            Document d = new Document();
+            // this field won't have any payloads
+            d.Add(NewTextField("f1", "this field has no payloads", Field.Store.NO));
+            // this field will have payloads in all docs, however not for all term positions,
+            // so this field is used to check if the DocumentWriter correctly enables the payloads bit
+            // even if only some term positions have payloads
+            d.Add(NewTextField("f2", "this field has payloads in all docs", Field.Store.NO));
+            d.Add(NewTextField("f2", "this field has payloads in all docs NO PAYLOAD", Field.Store.NO));
+            // this field is used to verify if the SegmentMerger enables payloads for a field if it has payloads
+            // enabled in only some documents
+            d.Add(NewTextField("f3", "this field has payloads in some docs", Field.Store.NO));
+            // only add payload data for field f2
+#pragma warning disable 612, 618
+            analyzer.SetPayloadData("f2", "somedata".GetBytes(IOUtils.CHARSET_UTF_8), 0, 1);
+#pragma warning restore 612, 618
+            writer.AddDocument(d);
+            // flush
+            writer.Dispose();
+
+            SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram));
+            FieldInfos fi = reader.FieldInfos;
+            Assert.IsFalse(fi.FieldInfo("f1").HasPayloads, "Payload field bit should not be set.");
+            Assert.IsTrue(fi.FieldInfo("f2").HasPayloads, "Payload field bit should be set.");
+            Assert.IsFalse(fi.FieldInfo("f3").HasPayloads, "Payload field bit should not be set.");
+            reader.Dispose();
+
+            // now we add another document which has payloads for field f3 and verify if the SegmentMerger
+            // enabled payloads for that field
+            analyzer = new PayloadAnalyzer(); // Clear payload state for each field
+            writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetOpenMode(OpenMode.CREATE));
+            d = new Document();
+            d.Add(NewTextField("f1", "this field has no payloads", Field.Store.NO));
+            d.Add(NewTextField("f2", "this field has payloads in all docs", Field.Store.NO));
+            d.Add(NewTextField("f2", "this field has payloads in all docs", Field.Store.NO));
+            d.Add(NewTextField("f3", "this field has payloads in some docs", Field.Store.NO));
+            // add payload data for field f2 and f3
+#pragma warning disable 612, 618
+            analyzer.SetPayloadData("f2", "somedata".GetBytes(IOUtils.CHARSET_UTF_8), 0, 1);
+            analyzer.SetPayloadData("f3", "somedata".GetBytes(IOUtils.CHARSET_UTF_8), 0, 3);
+#pragma warning restore 612, 618
+            writer.AddDocument(d);
+
+            // force merge
+            writer.ForceMerge(1);
+            // flush
+            writer.Dispose();
+
+            reader = GetOnlySegmentReader(DirectoryReader.Open(ram));
+            fi = reader.FieldInfos;
+            Assert.IsFalse(fi.FieldInfo("f1").HasPayloads, "Payload field bit should not be set.");
+            Assert.IsTrue(fi.FieldInfo("f2").HasPayloads, "Payload field bit should be set.");
+            Assert.IsTrue(fi.FieldInfo("f3").HasPayloads, "Payload field bit should be set.");
+            reader.Dispose();
+            ram.Dispose();
+        }
+
+        // Tests if payloads are correctly stored and loaded using both RamDirectory and FSDirectory
+        [Test]
+        public virtual void TestPayloadsEncoding()
+        {
+            Directory dir = NewDirectory();
+            PerformTest(dir);
+            dir.Dispose();
+        }
+
+        // builds an index with payloads in the given Directory and performs
+        // different tests to verify the payload encoding
+        private void PerformTest(Directory dir)
+        {
+            PayloadAnalyzer analyzer = new PayloadAnalyzer();
+            IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetOpenMode(OpenMode.CREATE).SetMergePolicy(NewLogMergePolicy()));
+
+            // should be in sync with value in TermInfosWriter
+            const int skipInterval = 16;
+
+            const int numTerms = 5;
+            const string fieldName = "f1";
+
+            int numDocs = skipInterval + 1;
+            // create content for the test documents with just a few terms
+            Term[] terms = GenerateTerms(fieldName, numTerms);
+            StringBuilder sb = new StringBuilder();
+            for (int i = 0; i < terms.Length; i++)
+            {
+                sb.Append(terms[i].Text());
+                sb.Append(" ");
+            }
+            string content = sb.ToString();
+
+            int payloadDataLength = numTerms * numDocs * 2 + numTerms * numDocs * (numDocs - 1) / 2;
+            var payloadData = GenerateRandomData(payloadDataLength);
+
+            Document d = new Document();
+            d.Add(NewTextField(fieldName, content, Field.Store.NO));
+            // add the same document multiple times to have the same payload lengths for all
+            // occurrences within two consecutive skip intervals
+            int offset = 0;
+            for (int i = 0; i < 2 * numDocs; i++)
+            {
+                analyzer = new PayloadAnalyzer(fieldName, payloadData, offset, 1);
+                offset += numTerms;
+                writer.AddDocument(d, analyzer);
+            }
+
+            // make sure we create more than one segment to test merging
+            writer.Commit();
+
+            // now we make sure to have different payload lengths next at the next skip point
+            for (int i = 0; i < numDocs; i++)
+            {
+                analyzer = new PayloadAnalyzer(fieldName, payloadData, offset, i);
+                offset += i * numTerms;
+                writer.AddDocument(d, analyzer);
+            }
+
+            writer.ForceMerge(1);
+            // flush
+            writer.Dispose();
+
+            /*
+             * Verify the index
+             * first we test if all payloads are stored correctly
+             */
+            IndexReader reader = DirectoryReader.Open(dir);
+
+            var verifyPayloadData = new byte[payloadDataLength];
+            offset = 0;
+            var tps = new DocsAndPositionsEnum[numTerms];
+            for (int i = 0; i < numTerms; i++)
+            {
+                tps[i] = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), terms[i].Field, new BytesRef(terms[i].Text()));
+            }
+
+            while (tps[0].NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
+            {
+                for (int i = 1; i < numTerms; i++)
+                {
+                    tps[i].NextDoc();
+                }
+                int freq = tps[0].Freq;
+
+                for (int i = 0; i < freq; i++)
+                {
+                    for (int j = 0; j < numTerms; j++)
+                    {
+                        tps[j].NextPosition();
+                        BytesRef br = tps[j].GetPayload();
+                        if (br != null)
+                        {
+                            Array.Copy(br.Bytes, br.Offset, verifyPayloadData, offset, br.Length);
+                            offset += br.Length;
+                        }
+                    }
+                }
+            }
+
+            AssertByteArrayEquals(payloadData, verifyPayloadData);
+
+            /*
+             *  test lazy skipping
+             */
+            DocsAndPositionsEnum tp = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), terms[0].Field, new BytesRef(terms[0].Text()));
+            tp.NextDoc();
+            tp.NextPosition();
+            // NOTE: prior rev of this test was failing to first
+            // call next here:
+            tp.NextDoc();
+            // now we don't read this payload
+            tp.NextPosition();
+            BytesRef payload = tp.GetPayload();
+            Assert.AreEqual(1, payload.Length, "Wrong payload length.");
+            Assert.AreEqual(payload.Bytes[payload.Offset], payloadData[numTerms]);
+            tp.NextDoc();
+            tp.NextPosition();
+
+            // we don't read this payload and skip to a different document
+            tp.Advance(5);
+            tp.NextPosition();
+            payload = tp.GetPayload();
+            Assert.AreEqual(1, payload.Length, "Wrong payload length.");
+            Assert.AreEqual(payload.Bytes[payload.Offset], payloadData[5 * numTerms]);
+
+            /*
+             * Test different lengths at skip points
+             */
+            tp = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), terms[1].Field, new BytesRef(terms[1].Text()));
+            tp.NextDoc();
+            tp.NextPosition();
+            Assert.AreEqual(1, tp.GetPayload().Length, "Wrong payload length.");
+            tp.Advance(skipInterval - 1);
+            tp.NextPosition();
+            Assert.AreEqual(1, tp.GetPayload().Length, "Wrong payload length.");
+            tp.Advance(2 * skipInterval - 1);
+            tp.NextPosition();
+            Assert.AreEqual(1, tp.GetPayload().Length, "Wrong payload length.");
+            tp.Advance(3 * skipInterval - 1);
+            tp.NextPosition();
+            Assert.AreEqual(3 * skipInterval - 2 * numDocs - 1, tp.GetPayload().Length, "Wrong payload length.");
+
+            reader.Dispose();
+
+            // test long payload
+            analyzer = new PayloadAnalyzer();
+            writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetOpenMode(OpenMode.CREATE));
+            string singleTerm = "lucene";
+
+            d = new Document();
+            d.Add(NewTextField(fieldName, singleTerm, Field.Store.NO));
+            // add a payload whose length is greater than the buffer size of BufferedIndexOutput
+            payloadData = GenerateRandomData(2000);
+            analyzer.SetPayloadData(fieldName, payloadData, 100, 1500);
+            writer.AddDocument(d);
+
+            writer.ForceMerge(1);
+            // flush
+            writer.Dispose();
+
+            reader = DirectoryReader.Open(dir);
+            tp = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), fieldName, new BytesRef(singleTerm));
+            tp.NextDoc();
+            tp.NextPosition();
+
+            BytesRef bref = tp.GetPayload();
+            verifyPayloadData = new byte[bref.Length];
+            var portion = new byte[1500];
+            Array.Copy(payloadData, 100, portion, 0, 1500);
+
+            AssertByteArrayEquals(portion, bref.Bytes, bref.Offset, bref.Length);
+            reader.Dispose();
+        }
+
+#pragma warning disable 612, 618
+        internal static readonly Encoding Utf8 = IOUtils.CHARSET_UTF_8;
+#pragma warning restore 612, 618
+
+        private void GenerateRandomData(byte[] data)
+        {
+            // this test needs the random data to be valid unicode
+            string s = TestUtil.RandomFixedByteLengthUnicodeString(Random(), data.Length);
+            var b = s.GetBytes(Utf8);
+            Debug.Assert(b.Length == data.Length);
+            System.Buffer.BlockCopy(b, 0, data, 0, b.Length);
+        }
+
+        private byte[] GenerateRandomData(int n)
+        {
+            var data = new byte[n];
+            GenerateRandomData(data);
+            return data;
+        }
+
+        private Term[] GenerateTerms(string fieldName, int n)
+        {
+            int maxDigits = (int)(Math.Log(n) / Math.Log(10));
+            Term[] terms = new Term[n];
+            StringBuilder sb = new StringBuilder();
+            for (int i = 0; i < n; i++)
+            {
+                sb.Length = 0;
+                sb.Append("t");
+                int zeros = maxDigits - (int)(Math.Log(i) / Math.Log(10));
+                for (int j = 0; j < zeros; j++)
+                {
+                    sb.Append("0");
+                }
+                sb.Append(i);
+                terms[i] = new Term(fieldName, sb.ToString());
+            }
+            return terms;
+        }
+
+        internal virtual void AssertByteArrayEquals(byte[] b1, byte[] b2)
+        {
+            if (b1.Length != b2.Length)
+            {
+                Assert.Fail("Byte arrays have different lengths: " + b1.Length + ", " + b2.Length);
+            }
+
+            for (int i = 0; i < b1.Length; i++)
+            {
+                if (b1[i] != b2[i])
+                {
+                    Assert.Fail("Byte arrays different at index " + i + ": " + b1[i] + ", " + b2[i]);
+                }
+            }
+        }
+
+        internal virtual void AssertByteArrayEquals(byte[] b1, byte[] b2, int b2offset, int b2length)
+        {
+            if (b1.Length != b2length)
+            {
+                Assert.Fail("Byte arrays have different lengths: " + b1.Length + ", " + b2length);
+            }
+
+            for (int i = 0; i < b1.Length; i++)
+            {
+                if (b1[i] != b2[b2offset + i])
+                {
+                    Assert.Fail("Byte arrays different at index " + i + ": " + b1[i] + ", " + b2[b2offset + i]);
+                }
+            }
+        }
+
+        /// <summary>
+        /// this Analyzer uses an WhitespaceTokenizer and PayloadFilter.
+        /// </summary>
+        private class PayloadAnalyzer : Analyzer
+        {
+            internal readonly IDictionary<string, PayloadData> FieldToData = new Dictionary<string, PayloadData>();
+
+            public PayloadAnalyzer()
+                : base(PER_FIELD_REUSE_STRATEGY)
+            {
+            }
+
+            public PayloadAnalyzer(string field, byte[] data, int offset, int length)
+                : base(PER_FIELD_REUSE_STRATEGY)
+            {
+                SetPayloadData(field, data, offset, length);
+            }
+
+            internal virtual void SetPayloadData(string field, byte[] data, int offset, int length)
+            {
+                FieldToData[field] = new PayloadData(data, offset, length);
+            }
+
+            protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+            {
+                PayloadData payload;
+                FieldToData.TryGetValue(fieldName, out payload);
+                Tokenizer ts = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+                TokenStream tokenStream = (payload != null) ? (TokenStream)new PayloadFilter(ts, payload.Data, payload.Offset, payload.Length) : ts;
+                return new TokenStreamComponents(ts, tokenStream);
+            }
+
+            internal class PayloadData
+            {
+                internal byte[] Data;
+                internal int Offset;
+                internal int Length;
+
+                internal PayloadData(byte[] data, int offset, int length)
+                {
+                    this.Data = data;
+                    this.Offset = offset;
+                    this.Length = length;
+                }
+            }
+        }
+
+        /// <summary>
+        /// this Filter adds payloads to the tokens.
+        /// </summary>
+        private class PayloadFilter : TokenFilter
+        {
+            internal byte[] Data;
+            internal int Length;
+            internal int Offset;
+            internal int StartOffset;
+            internal IPayloadAttribute PayloadAtt;
+            internal ICharTermAttribute TermAttribute;
+
+            public PayloadFilter(TokenStream @in, byte[] data, int offset, int length)
+                : base(@in)
+            {
+                this.Data = data;
+                this.Length = length;
+                this.Offset = offset;
+                this.StartOffset = offset;
+                PayloadAtt = AddAttribute<IPayloadAttribute>();
+                TermAttribute = AddAttribute<ICharTermAttribute>();
+            }
+
+            public sealed override bool IncrementToken()
+            {
+                bool hasNext = m_input.IncrementToken();
+                if (!hasNext)
+                {
+                    return false;
+                }
+
+                // Some values of the same field are to have payloads and others not
+                if (Offset + Length <= Data.Length && !TermAttribute.ToString().EndsWith("NO PAYLOAD"))
+                {
+                    BytesRef p = new BytesRef(Data, Offset, Length);
+                    PayloadAtt.Payload = p;
+                    Offset += Length;
+                }
+                else
+                {
+                    PayloadAtt.Payload = null;
+                }
+
+                return true;
+            }
+
+            public override void Reset()
+            {
+                base.Reset();
+                this.Offset = StartOffset;
+            }
+        }
+
+        [Test]
+        public virtual void TestThreadSafety()
+        {
+            const int numThreads = 5;
+            int numDocs = AtLeast(50);
+            ByteArrayPool pool = new ByteArrayPool(numThreads, 5);
+
+            Directory dir = NewDirectory();
+            IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+            const string field = "test";
+
+            ThreadClass[] ingesters = new ThreadClass[numThreads];
+            for (int i = 0; i < numThreads; i++)
+            {
+                ingesters[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, pool, writer, field);
+                ingesters[i].Start();
+            }
+
+            for (int i = 0; i < numThreads; i++)
+            {
+                ingesters[i].Join();
+            }
+            writer.Dispose();
+            IndexReader reader = DirectoryReader.Open(dir);
+            TermsEnum terms = MultiFields.GetFields(reader).GetTerms(field).GetIterator(null);
+            IBits liveDocs = MultiFields.GetLiveDocs(reader);
+            DocsAndPositionsEnum tp = null;
+            while (terms.Next() != null)
+            {
+                string termText = terms.Term.Utf8ToString();
+                tp = terms.DocsAndPositions(liveDocs, tp);
+                while (tp.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
+                {
+                    int freq = tp.Freq;
+                    for (int i = 0; i < freq; i++)
+                    {
+                        tp.NextPosition();
+                        BytesRef payload = tp.GetPayload();
+                        Assert.AreEqual(termText, payload.Utf8ToString());
+                    }
+                }
+            }
+            reader.Dispose();
+            dir.Dispose();
+            Assert.AreEqual(pool.Count, numThreads);
+        }
+
+        private class ThreadAnonymousInnerClassHelper : ThreadClass
+        {
+            private readonly TestPayloads OuterInstance;
+
+            private int NumDocs;
+            private Lucene.Net.Index.TestPayloads.ByteArrayPool Pool;
+            private IndexWriter Writer;
+            private string Field;
+
+            public ThreadAnonymousInnerClassHelper(TestPayloads outerInstance, int numDocs, Lucene.Net.Index.TestPayloads.ByteArrayPool pool, IndexWriter writer, string field)
+            {
+                this.OuterInstance = outerInstance;
+                this.NumDocs = numDocs;
+                this.Pool = pool;
+                this.Writer = writer;
+                this.Field = field;
+            }
+
+            public override void Run()
+            {
+                try
+                {
+                    for (int j = 0; j < NumDocs; j++)
+                    {
+                        Document d = new Document();
+                        d.Add(new TextField(Field, new PoolingPayloadTokenStream(OuterInstance, Pool)));
+                        Writer.AddDocument(d);
+                    }
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e.ToString());
+                    Console.Write(e.StackTrace);
+                    Assert.Fail(e.ToString());
+                }
+            }
+        }
+
+        private class PoolingPayloadTokenStream : TokenStream
+        {
+            private readonly TestPayloads OuterInstance;
+
+            private byte[] Payload;
+            internal bool First;
+            internal ByteArrayPool Pool;
+            internal string Term;
+
+            internal ICharTermAttribute TermAtt;
+            internal IPayloadAttribute PayloadAtt;
+
+            internal PoolingPayloadTokenStream(TestPayloads outerInstance, ByteArrayPool pool)
+            {
+                this.OuterInstance = outerInstance;
+                this.Pool = pool;
+                Payload = pool.Get();
+                OuterInstance.GenerateRandomData(Payload);
+                Term = Encoding.UTF8.GetString((byte[])(Array)Payload);
+                First = true;
+                PayloadAtt = AddAttribute<IPayloadAttribute>();
+                TermAtt = AddAttribute<ICharTermAttribute>();
+            }
+
+            public sealed override bool IncrementToken()
+            {
+                if (!First)
+                {
+                    return false;
+                }
+                First = false;
+                ClearAttributes();
+                TermAtt.Append(Term);
+                PayloadAtt.Payload = new BytesRef(Payload);
+                return true;
+            }
+
+            public override void Dispose()
+            {
+                Pool.Release(Payload);
+            }
+        }
+
+        private class ByteArrayPool
+        {
+            internal readonly IList<byte[]> Pool;
+
+            internal ByteArrayPool(int capacity, int size)
+            {
+                Pool = new List<byte[]>();
+                for (int i = 0; i < capacity; i++)
+                {
+                    Pool.Add(new byte[size]);
+                }
+            }
+
+            internal virtual byte[] Get()
+            {
+                lock (this) // TODO use BlockingCollection / BCL datastructures instead
+                {
+                    var retArray = Pool[0];
+                    Pool.RemoveAt(0);
+                    return retArray;
+                }
+            }
+
+            internal virtual void Release(byte[] b)
+            {
+                lock (this)
+                {
+                    Pool.Add(b);
+                }
+            }
+
+            internal virtual int Count
+            {
+                get
+                {
+                    lock (this)
+                    {
+                        return Pool.Count;
+                    }
+                }
+            }
+        }
+
+        [Test]
+        public virtual void TestAcrossFields()
+        {
+            Directory dir = NewDirectory();
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, true), Similarity, TimeZone);
+            Document doc = new Document();
+            doc.Add(new TextField("hasMaybepayload", "here we go", Field.Store.YES));
+            writer.AddDocument(doc);
+            writer.Dispose();
+
+            writer = new RandomIndexWriter(Random(), dir, new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, true), Similarity, TimeZone);
+            doc = new Document();
+            doc.Add(new TextField("hasMaybepayload2", "here we go", Field.Store.YES));
+            writer.AddDocument(doc);
+            writer.AddDocument(doc);
+            writer.ForceMerge(1);
+            writer.Dispose();
+
+            dir.Dispose();
+        }
+
+        /// <summary>
+        /// some docs have payload att, some not </summary>
+        [Test]
+        public virtual void TestMixupDocs()
+        {
+            Directory dir = NewDirectory();
+            IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, null);
+            iwc.SetMergePolicy(NewLogMergePolicy());
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
+            Document doc = new Document();
+            Field field = new TextField("field", "", Field.Store.NO);
+            TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+            Assert.IsFalse(ts.HasAttribute<PayloadAttribute>());
+            field.SetTokenStream(ts);
+            doc.Add(field);
+            writer.AddDocument(doc);
+            Token withPayload = new Token("withPayload", 0, 11);
+            withPayload.Payload = new BytesRef("test");
+            ts = new CannedTokenStream(withPayload);
+            Assert.IsTrue(ts.HasAttribute<IPayloadAttribute>());
+            field.SetTokenStream(ts);
+            writer.AddDocument(doc);
+            ts = new MockTokenizer(new StringReader("another"), MockTokenizer.WHITESPACE, true);
+            Assert.IsFalse(ts.HasAttribute<PayloadAttribute>());
+            field.SetTokenStream(ts);
+            writer.AddDocument(doc);
+            DirectoryReader reader = writer.Reader;
+            AtomicReader sr = SlowCompositeReaderWrapper.Wrap(reader);
+            DocsAndPositionsEnum de = sr.TermPositionsEnum(new Term("field", "withPayload"));
+            de.NextDoc();
+            de.NextPosition();
+            Assert.AreEqual(new BytesRef("test"), de.GetPayload());
+            writer.Dispose();
+            reader.Dispose();
+            dir.Dispose();
+        }
+
+        /// <summary>
+        /// some field instances have payload att, some not </summary>
+        [Test]
+        public virtual void TestMixupMultiValued()
+        {
+            Directory dir = NewDirectory();
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
+            Document doc = new Document();
+            Field field = new TextField("field", "", Field.Store.NO);
+            TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+            Assert.IsFalse(ts.HasAttribute<PayloadAttribute>());
+            field.SetTokenStream(ts);
+            doc.Add(field);
+            Field field2 = new TextField("field", "", Field.Store.NO);
+            Token withPayload = new Token("withPayload", 0, 11);
+            withPayload.Payload = new BytesRef("test");
+            ts = new CannedTokenStream(withPayload);
+            Assert.IsTrue(ts.HasAttribute<IPayloadAttribute>());
+            field2.SetTokenStream(ts);
+            doc.Add(field2);
+            Field field3 = new TextField("field", "", Field.Store.NO);
+            ts = new MockTokenizer(new StringReader("nopayload"), MockTokenizer.WHITESPACE, true);
+            Assert.IsFalse(ts.HasAttribute<PayloadAttribute>());
+            field3.SetTokenStream(ts);
+            doc.Add(field3);
+            writer.AddDocument(doc);
+            DirectoryReader reader = writer.Reader;
+            SegmentReader sr = GetOnlySegmentReader(reader);
+            DocsAndPositionsEnum de = sr.TermPositionsEnum(new Term("field", "withPayload"));
+            de.NextDoc();
+            de.NextPosition();
+            Assert.AreEqual(new BytesRef("test"), de.GetPayload());
+            writer.Dispose();
+            reader.Dispose();
+            dir.Dispose();
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestPayloadsOnVectors.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestPayloadsOnVectors.cs b/src/Lucene.Net.Tests/Index/TestPayloadsOnVectors.cs
new file mode 100644
index 0000000..7e26232
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestPayloadsOnVectors.cs
@@ -0,0 +1,165 @@
+using Lucene.Net.Analysis.TokenAttributes;
+using System.Diagnostics;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+    using Lucene.Net.Randomized.Generators;
+    using NUnit.Framework;
+    using System.IO;
+    using BytesRef = Lucene.Net.Util.BytesRef;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using CannedTokenStream = Lucene.Net.Analysis.CannedTokenStream;
+    using Directory = Lucene.Net.Store.Directory;
+    using Document = Documents.Document;
+    using Field = Field;
+    using FieldType = FieldType;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using MockTokenizer = Lucene.Net.Analysis.MockTokenizer;
+    using TextField = TextField;
+    using Token = Lucene.Net.Analysis.Token;
+    using TokenStream = Lucene.Net.Analysis.TokenStream;
+
+    [SuppressCodecs("Lucene3x")]
+    [TestFixture]
+    public class TestPayloadsOnVectors : LuceneTestCase
+    {
+        /// <summary>
+        /// some docs have payload att, some not </summary>
+        [Test]
+        public virtual void TestMixupDocs()
+        {
+            Directory dir = NewDirectory();
+            IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+            iwc.SetMergePolicy(NewLogMergePolicy());
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, iwc);
+            Document doc = new Document();
+            FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+            customType.StoreTermVectors = true;
+            customType.StoreTermVectorPositions = true;
+            customType.StoreTermVectorPayloads = true;
+            customType.StoreTermVectorOffsets = Random().NextBoolean();
+            Field field = new Field("field", "", customType);
+            TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+            Assert.IsFalse(ts.HasAttribute<IPayloadAttribute>());
+            field.SetTokenStream(ts);
+            doc.Add(field);
+            writer.AddDocument(doc);
+
+            Token withPayload = new Token("withPayload", 0, 11);
+            withPayload.Payload = new BytesRef("test");
+            ts = new CannedTokenStream(withPayload);
+            Assert.IsTrue(ts.HasAttribute<IPayloadAttribute>());
+            field.SetTokenStream(ts);
+            writer.AddDocument(doc);
+
+            ts = new MockTokenizer(new StringReader("another"), MockTokenizer.WHITESPACE, true);
+            Assert.IsFalse(ts.HasAttribute<IPayloadAttribute>());
+            field.SetTokenStream(ts);
+            writer.AddDocument(doc);
+
+            DirectoryReader reader = writer.Reader;
+            Terms terms = reader.GetTermVector(1, "field");
+            Debug.Assert(terms != null);
+            TermsEnum termsEnum = terms.GetIterator(null);
+            Assert.IsTrue(termsEnum.SeekExact(new BytesRef("withPayload")));
+            DocsAndPositionsEnum de = termsEnum.DocsAndPositions(null, null);
+            Assert.AreEqual(0, de.NextDoc());
+            Assert.AreEqual(0, de.NextPosition());
+            Assert.AreEqual(new BytesRef("test"), de.GetPayload());
+            writer.Dispose();
+            reader.Dispose();
+            dir.Dispose();
+        }
+
+        /// <summary>
+        /// some field instances have payload att, some not </summary>
+        [Test]
+        public virtual void TestMixupMultiValued()
+        {
+            Directory dir = NewDirectory();
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
+            Document doc = new Document();
+            FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+            customType.StoreTermVectors = true;
+            customType.StoreTermVectorPositions = true;
+            customType.StoreTermVectorPayloads = true;
+            customType.StoreTermVectorOffsets = Random().NextBoolean();
+            Field field = new Field("field", "", customType);
+            TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+            Assert.IsFalse(ts.HasAttribute<IPayloadAttribute>());
+            field.SetTokenStream(ts);
+            doc.Add(field);
+            Field field2 = new Field("field", "", customType);
+            Token withPayload = new Token("withPayload", 0, 11);
+            withPayload.Payload = new BytesRef("test");
+            ts = new CannedTokenStream(withPayload);
+            Assert.IsTrue(ts.HasAttribute<IPayloadAttribute>());
+            field2.SetTokenStream(ts);
+            doc.Add(field2);
+            Field field3 = new Field("field", "", customType);
+            ts = new MockTokenizer(new StringReader("nopayload"), MockTokenizer.WHITESPACE, true);
+            Assert.IsFalse(ts.HasAttribute<IPayloadAttribute>());
+            field3.SetTokenStream(ts);
+            doc.Add(field3);
+            writer.AddDocument(doc);
+            DirectoryReader reader = writer.Reader;
+            Terms terms = reader.GetTermVector(0, "field");
+            Debug.Assert(terms != null);
+            TermsEnum termsEnum = terms.GetIterator(null);
+            Assert.IsTrue(termsEnum.SeekExact(new BytesRef("withPayload")));
+            DocsAndPositionsEnum de = termsEnum.DocsAndPositions(null, null);
+            Assert.AreEqual(0, de.NextDoc());
+            Assert.AreEqual(3, de.NextPosition());
+            Assert.AreEqual(new BytesRef("test"), de.GetPayload());
+            writer.Dispose();
+            reader.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestPayloadsWithoutPositions()
+        {
+            Directory dir = NewDirectory();
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
+            Document doc = new Document();
+            FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+            customType.StoreTermVectors = true;
+            customType.StoreTermVectorPositions = false;
+            customType.StoreTermVectorPayloads = true;
+            customType.StoreTermVectorOffsets = Random().NextBoolean();
+            doc.Add(new Field("field", "foo", customType));
+            try
+            {
+                writer.AddDocument(doc);
+                Assert.Fail();
+            }
+#pragma warning disable 168
+            catch (System.ArgumentException expected)
+#pragma warning restore 168
+            {
+                // expected
+            }
+            writer.Dispose();
+            dir.Dispose();
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestPerSegmentDeletes.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestPerSegmentDeletes.cs b/src/Lucene.Net.Tests/Index/TestPerSegmentDeletes.cs
new file mode 100644
index 0000000..ce7e767
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestPerSegmentDeletes.cs
@@ -0,0 +1,318 @@
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Index
+{
+    using Lucene.Net.Support;
+    using NUnit.Framework;
+    using ArrayUtil = Lucene.Net.Util.ArrayUtil;
+    using IBits = Lucene.Net.Util.IBits;
+    using BytesRef = Lucene.Net.Util.BytesRef;
+    using Directory = Lucene.Net.Store.Directory;
+    using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+    using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+    using TestUtil = Lucene.Net.Util.TestUtil;
+
+    [TestFixture]
+    public class TestPerSegmentDeletes : LuceneTestCase
+    {
+        [Test]
+        public virtual void TestDeletes1()
+        {
+            //IndexWriter.debug2 = System.out;
+            Directory dir = new MockDirectoryWrapper(new Random(Random().Next()), new RAMDirectory());
+            IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+            iwc.SetMergeScheduler(new SerialMergeScheduler());
+            iwc.SetMaxBufferedDocs(5000);
+            iwc.SetRAMBufferSizeMB(100);
+            RangeMergePolicy fsmp = new RangeMergePolicy(this, false);
+            iwc.SetMergePolicy(fsmp);
+            IndexWriter writer = new IndexWriter(dir, iwc);
+            for (int x = 0; x < 5; x++)
+            {
+                writer.AddDocument(DocHelper.CreateDocument(x, "1", 2));
+                //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
+            }
+            //System.out.println("commit1");
+            writer.Commit();
+            Assert.AreEqual(1, writer.SegmentCount);
+            for (int x = 5; x < 10; x++)
+            {
+                writer.AddDocument(DocHelper.CreateDocument(x, "2", 2));
+                //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
+            }
+            //System.out.println("commit2");
+            writer.Commit();
+            Assert.AreEqual(2, writer.SegmentCount);
+
+            for (int x = 10; x < 15; x++)
+            {
+                writer.AddDocument(DocHelper.CreateDocument(x, "3", 2));
+                //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
+            }
+
+            writer.DeleteDocuments(new Term("id", "1"));
+
+            writer.DeleteDocuments(new Term("id", "11"));
+
+            // flushing without applying deletes means
+            // there will still be deletes in the segment infos
+            writer.Flush(false, false);
+            Assert.IsTrue(writer.bufferedUpdatesStream.Any());
+
+            // get reader flushes pending deletes
+            // so there should not be anymore
+            IndexReader r1 = writer.Reader;
+            Assert.IsFalse(writer.bufferedUpdatesStream.Any());
+            r1.Dispose();
+
+            // delete id:2 from the first segment
+            // merge segments 0 and 1
+            // which should apply the delete id:2
+            writer.DeleteDocuments(new Term("id", "2"));
+            writer.Flush(false, false);
+            fsmp = (RangeMergePolicy)writer.Config.MergePolicy;
+            fsmp.DoMerge = true;
+            fsmp.Start = 0;
+            fsmp.Length = 2;
+            writer.MaybeMerge();
+
+            Assert.AreEqual(2, writer.SegmentCount);
+
+            // id:2 shouldn't exist anymore because
+            // it's been applied in the merge and now it's gone
+            IndexReader r2 = writer.Reader;
+            int[] id2docs = ToDocsArray(new Term("id", "2"), null, r2);
+            Assert.IsTrue(id2docs == null);
+            r2.Dispose();
+
+            /*
+            /// // added docs are in the ram buffer
+            /// for (int x = 15; x < 20; x++) {
+            ///  writer.AddDocument(TestIndexWriterReader.CreateDocument(x, "4", 2));
+            ///  System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
+            /// }
+            /// Assert.IsTrue(writer.numRamDocs() > 0);
+            /// // delete from the ram buffer
+            /// writer.DeleteDocuments(new Term("id", Integer.toString(13)));
+            ///
+            /// Term id3 = new Term("id", Integer.toString(3));
+            ///
+            /// // delete from the 1st segment
+            /// writer.DeleteDocuments(id3);
+            ///
+            /// Assert.IsTrue(writer.numRamDocs() > 0);
+            ///
+            /// //System.out
+            /// //    .println("segdels1:" + writer.docWriter.deletesToString());
+            ///
+            /// //Assert.IsTrue(writer.docWriter.segmentDeletes.Size() > 0);
+            ///
+            /// // we cause a merge to happen
+            /// fsmp.doMerge = true;
+            /// fsmp.start = 0;
+            /// fsmp.Length = 2;
+            /// System.out.println("maybeMerge "+writer.SegmentInfos);
+            ///
+            /// SegmentInfo info0 = writer.SegmentInfos.Info(0);
+            /// SegmentInfo info1 = writer.SegmentInfos.Info(1);
+            ///
+            /// writer.MaybeMerge();
+            /// System.out.println("maybeMerge after "+writer.SegmentInfos);
+            /// // there should be docs in RAM
+            /// Assert.IsTrue(writer.numRamDocs() > 0);
+            ///
+            /// // assert we've merged the 1 and 2 segments
+            /// // and still have a segment leftover == 2
+            /// Assert.AreEqual(2, writer.SegmentInfos.Size());
+            /// Assert.IsFalse(segThere(info0, writer.SegmentInfos));
+            /// Assert.IsFalse(segThere(info1, writer.SegmentInfos));
+            ///
+            /// //System.out.println("segdels2:" + writer.docWriter.deletesToString());
+            ///
+            /// //Assert.IsTrue(writer.docWriter.segmentDeletes.Size() > 0);
+            ///
+            /// IndexReader r = writer.GetReader();
+            /// IndexReader r1 = r.getSequentialSubReaders()[0];
+            /// printDelDocs(r1.GetLiveDocs());
+            /// int[] docs = toDocsArray(id3, null, r);
+            /// System.out.println("id3 docs:"+Arrays.toString(docs));
+            /// // there shouldn't be any docs for id:3
+            /// Assert.IsTrue(docs == null);
+            /// r.Dispose();
+            ///
+            /// part2(writer, fsmp);
+            ///
+            */
+            // System.out.println("segdels2:"+writer.docWriter.segmentDeletes.toString());
+            //System.out.println("close");
+            writer.Dispose();
+            dir.Dispose();
+        }
+
+        /// <summary>
+        /// static boolean hasPendingDeletes(SegmentInfos infos) {
+        ///  for (SegmentInfo info : infos) {
+        ///    if (info.deletes.Any()) {
+        ///      return true;
+        ///    }
+        ///  }
+        ///  return false;
+        /// }
+        ///
+        /// </summary>
+        internal virtual void Part2(IndexWriter writer, RangeMergePolicy fsmp)
+        {
+            for (int x = 20; x < 25; x++)
+            {
+                writer.AddDocument(DocHelper.CreateDocument(x, "5", 2));
+                //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
+            }
+            writer.Flush(false, false);
+            for (int x = 25; x < 30; x++)
+            {
+                writer.AddDocument(DocHelper.CreateDocument(x, "5", 2));
+                //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
+            }
+            writer.Flush(false, false);
+
+            //System.out.println("infos3:"+writer.SegmentInfos);
+
+            Term delterm = new Term("id", "8");
+            writer.DeleteDocuments(delterm);
+            //System.out.println("segdels3:" + writer.docWriter.deletesToString());
+
+            fsmp.DoMerge = true;
+            fsmp.Start = 1;
+            fsmp.Length = 2;
+            writer.MaybeMerge();
+
+            // deletes for info1, the newly created segment from the
+            // merge should have no deletes because they were applied in
+            // the merge
+            //SegmentInfo info1 = writer.SegmentInfos.Info(1);
+            //Assert.IsFalse(exists(info1, writer.docWriter.segmentDeletes));
+
+            //System.out.println("infos4:"+writer.SegmentInfos);
+            //System.out.println("segdels4:" + writer.docWriter.deletesToString());
+        }
+
+        internal virtual bool SegThere(SegmentCommitInfo info, SegmentInfos infos)
+        {
+            foreach (SegmentCommitInfo si in infos.Segments)
+            {
+                if (si.Info.Name.Equals(info.Info.Name))
+                {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        public static void PrintDelDocs(IBits bits)
+        {
+            if (bits == null)
+            {
+                return;
+            }
+            for (int x = 0; x < bits.Length; x++)
+            {
+                Console.WriteLine(x + ":" + bits.Get(x));
+            }
+        }
+
+        public virtual int[] ToDocsArray(Term term, IBits bits, IndexReader reader)
+        {
+            Fields fields = MultiFields.GetFields(reader);
+            Terms cterms = fields.GetTerms(term.Field);
+            TermsEnum ctermsEnum = cterms.GetIterator(null);
+            if (ctermsEnum.SeekExact(new BytesRef(term.Text())))
+            {
+                DocsEnum docsEnum = TestUtil.Docs(Random(), ctermsEnum, bits, null, DocsEnum.FLAG_NONE);
+                return ToArray(docsEnum);
+            }
+            return null;
+        }
+
+        public static int[] ToArray(DocsEnum docsEnum)
+        {
+            IList<int?> docs = new List<int?>();
+            while (docsEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
+            {
+                int docID = docsEnum.DocID;
+                docs.Add(docID);
+            }
+            return ArrayUtil.ToInt32Array(docs);
+        }
+
+        public class RangeMergePolicy : MergePolicy
+        {
+            private readonly TestPerSegmentDeletes OuterInstance;
+
+            internal bool DoMerge = false;
+            internal int Start;
+            internal int Length;
+
+            internal readonly bool UseCompoundFile_Renamed;
+
+            internal RangeMergePolicy(TestPerSegmentDeletes outerInstance, bool useCompoundFile)
+            {
+                this.OuterInstance = outerInstance;
+                this.UseCompoundFile_Renamed = useCompoundFile;
+            }
+
+            public override void Dispose()
+            {
+            }
+
+            public override MergeSpecification FindMerges(MergeTrigger? mergeTrigger, SegmentInfos segmentInfos)
+            {
+                MergeSpecification ms = new MergeSpecification();
+                if (DoMerge)
+                {
+                    OneMerge om = new OneMerge(segmentInfos.AsList().SubList(Start, Start + Length));
+                    ms.Add(om);
+                    DoMerge = false;
+                    return ms;
+                }
+                return null;
+            }
+
+            public override MergeSpecification FindForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount, IDictionary<SegmentCommitInfo, bool?> segmentsToMerge)
+            {
+                return null;
+            }
+
+            public override MergeSpecification FindForcedDeletesMerges(SegmentInfos segmentInfos)
+            {
+                return null;
+            }
+
+            public override bool UseCompoundFile(SegmentInfos segments, SegmentCommitInfo newSegment)
+            {
+                return UseCompoundFile_Renamed;
+            }
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestPersistentSnapshotDeletionPolicy.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestPersistentSnapshotDeletionPolicy.cs b/src/Lucene.Net.Tests/Index/TestPersistentSnapshotDeletionPolicy.cs
new file mode 100644
index 0000000..84c7a59
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestPersistentSnapshotDeletionPolicy.cs
@@ -0,0 +1,260 @@
+using System.Diagnostics;
+
+namespace Lucene.Net.Index
+{
+    using NUnit.Framework;
+    using System;
+    using System.IO;
+    using Util;
+    using Directory = Lucene.Net.Store.Directory;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements. See the NOTICE file distributed with this
+         * work for additional information regarding copyright ownership. The ASF
+         * licenses this file to You under the Apache License, Version 2.0 (the
+         * "License"); you may not use this file except in compliance with the License.
+         * You may obtain a copy of the License at
+         *
+         * http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+         * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+         * License for the specific language governing permissions and limitations under
+         * the License.
+         */
+
+    using Document = Documents.Document;
+    using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+
+    [TestFixture]
+    public class TestPersistentSnapshotDeletionPolicy : TestSnapshotDeletionPolicy
+    {
+        [SetUp]
+        public override void SetUp()
+        {
+            base.SetUp();
+        }
+
+        [TearDown]
+        public override void TearDown()
+        {
+            base.TearDown();
+        }
+
+        private SnapshotDeletionPolicy GetDeletionPolicy(Directory dir)
+        {
+            return new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE);
+        }
+
+        [Test]
+        public virtual void TestExistingSnapshots()
+        {
+            int numSnapshots = 3;
+            MockDirectoryWrapper dir = NewMockDirectory();
+            IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), GetDeletionPolicy(dir)));
+            PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
+            Assert.IsNull(psdp.LastSaveFile);
+            PrepareIndexAndSnapshots(psdp, writer, numSnapshots);
+            Assert.IsNotNull(psdp.LastSaveFile);
+            writer.Dispose();
+
+            // Make sure only 1 save file exists:
+            int count = 0;
+            foreach (string file in dir.ListAll())
+            {
+                if (file.StartsWith(PersistentSnapshotDeletionPolicy.SNAPSHOTS_PREFIX))
+                {
+                    count++;
+                }
+            }
+            Assert.AreEqual(1, count);
+
+            // Make sure we fsync:
+            dir.Crash();
+            dir.ClearCrash();
+
+            // Re-initialize and verify snapshots were persisted
+            psdp = new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.APPEND);
+
+            writer = new IndexWriter(dir, GetConfig(Random(), psdp));
+            psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
+
+            Assert.AreEqual(numSnapshots, psdp.GetSnapshots().Count);
+            Assert.AreEqual(numSnapshots, psdp.SnapshotCount);
+            AssertSnapshotExists(dir, psdp, numSnapshots, false);
+
+            writer.AddDocument(new Document());
+            writer.Commit();
+            Snapshots.Add(psdp.Snapshot());
+            Assert.AreEqual(numSnapshots + 1, psdp.GetSnapshots().Count);
+            Assert.AreEqual(numSnapshots + 1, psdp.SnapshotCount);
+            AssertSnapshotExists(dir, psdp, numSnapshots + 1, false);
+
+            writer.Dispose();
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestNoSnapshotInfos()
+        {
+            Directory dir = NewDirectory();
+            new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE);
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestMissingSnapshots()
+        {
+            Directory dir = NewDirectory();
+            try
+            {
+                new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.APPEND);
+                Assert.Fail("did not hit expected exception");
+            }
+#pragma warning disable 168
+            catch (InvalidOperationException ise)
+#pragma warning restore 168
+            {
+                // expected
+            }
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestExceptionDuringSave()
+        {
+            MockDirectoryWrapper dir = NewMockDirectory();
+            dir.FailOn(new FailureAnonymousInnerClassHelper(this, dir));
+            IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE_OR_APPEND)));
+            writer.AddDocument(new Document());
+            writer.Commit();
+
+            PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
+            try
+            {
+                psdp.Snapshot();
+            }
+            catch (IOException ioe)
+            {
+                if (ioe.Message.Equals("now fail on purpose"))
+                {
+                    // ok
+                }
+                else
+                {
+                    throw ioe;
+                }
+            }
+            Assert.AreEqual(0, psdp.SnapshotCount);
+            writer.Dispose();
+            Assert.AreEqual(1, DirectoryReader.ListCommits(dir).Count);
+            dir.Dispose();
+        }
+
+        private class FailureAnonymousInnerClassHelper : MockDirectoryWrapper.Failure
+        {
+            private readonly TestPersistentSnapshotDeletionPolicy OuterInstance;
+
+            private MockDirectoryWrapper Dir;
+
+            public FailureAnonymousInnerClassHelper(TestPersistentSnapshotDeletionPolicy outerInstance, MockDirectoryWrapper dir)
+            {
+                this.OuterInstance = outerInstance;
+                this.Dir = dir;
+            }
+
+            public override void Eval(MockDirectoryWrapper dir)
+            {
+                /*typeof(PersistentSnapshotDeletionPolicy).Name.Equals(frame.GetType().Name) && */
+                if (StackTraceHelper.DoesStackTraceContainMethod("Persist"))
+                {
+                    throw new IOException("now fail on purpose");
+                }
+            }
+        }
+
+        [Test]
+        public virtual void TestSnapshotRelease()
+        {
+            Directory dir = NewDirectory();
+            IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), GetDeletionPolicy(dir)));
+            PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
+            PrepareIndexAndSnapshots(psdp, writer, 1);
+            writer.Dispose();
+
+            psdp.Release(Snapshots[0]);
+
+            psdp = new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.APPEND);
+            Assert.AreEqual(0, psdp.SnapshotCount, "Should have no snapshots !");
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestSnapshotReleaseByGeneration()
+        {
+            Directory dir = NewDirectory();
+            IndexWriter writer = new IndexWriter(dir, GetConfig(Random(), GetDeletionPolicy(dir)));
+            PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
+            PrepareIndexAndSnapshots(psdp, writer, 1);
+            writer.Dispose();
+
+            psdp.Release(Snapshots[0].Generation);
+
+            psdp = new PersistentSnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.APPEND);
+            Assert.AreEqual(0, psdp.SnapshotCount, "Should have no snapshots !");
+            dir.Dispose();
+        }
+
+
+        #region TestSnapshotDeletionPolicy
+        // LUCENENET NOTE: Tests in a base class are not pulled into the correct
+        // context in Visual Studio. This fixes that with the minimum amount of code necessary
+        // to run them in the correct context without duplicating all of the tests.
+
+        [Test]
+        public override void TestSnapshotDeletionPolicy_Mem()
+        {
+            base.TestSnapshotDeletionPolicy_Mem();
+        }
+
+        [Test]
+        public override void TestBasicSnapshots()
+        {
+            base.TestBasicSnapshots();
+        }
+
+        [Test]
+        public override void TestMultiThreadedSnapshotting()
+        {
+            base.TestMultiThreadedSnapshotting();
+        }
+
+        [Test]
+        public override void TestRollbackToOldSnapshot()
+        {
+            base.TestRollbackToOldSnapshot();
+        }
+
+        [Test]
+        public override void TestReleaseSnapshot()
+        {
+            base.TestReleaseSnapshot();
+        }
+
+        [Test]
+        public override void TestSnapshotLastCommitTwice()
+        {
+            base.TestSnapshotLastCommitTwice();
+        }
+
+        [Test]
+        public override void TestMissingCommits()
+        {
+            base.TestMissingCommits();
+        }
+
+        #endregion
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestPostingsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestPostingsFormat.cs b/src/Lucene.Net.Tests/Index/TestPostingsFormat.cs
new file mode 100644
index 0000000..20c6b07
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestPostingsFormat.cs
@@ -0,0 +1,95 @@
+using NUnit.Framework;
+
+namespace Lucene.Net.Index
+{
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+
+    using Codec = Lucene.Net.Codecs.Codec;
+
+    /// <summary>
+    /// Tests the codec configuration defined by LuceneTestCase randomly
+    ///  (typically a mix across different fields).
+    /// </summary>
+    [TestFixture]
+    public class TestPostingsFormat : BasePostingsFormatTestCase
+    {
+        protected override Codec Codec
+        {
+            get
+            {
+                return Codec.Default;
+            }
+        }
+
+        [Test]
+        public override void TestMergeStability()
+        {
+            AssumeTrue("The MockRandom PF randomizes content on the fly, so we can't check it", false);
+        }
+
+
+        #region BasePostingsFormatTestCase
+        // LUCENENET NOTE: Tests in an abstract base class are not pulled into the correct
+        // context in Visual Studio. This fixes that with the minimum amount of code necessary
+        // to run them in the correct context without duplicating all of the tests.
+
+        [Test]
+        public override void TestDocsOnly()
+        {
+            base.TestDocsOnly();
+        }
+
+        [Test]
+        public override void TestDocsAndFreqs()
+        {
+            base.TestDocsAndFreqs();
+        }
+
+        [Test]
+        public override void TestDocsAndFreqsAndPositions()
+        {
+            base.TestDocsAndFreqsAndPositions();
+        }
+
+        [Test]
+        public override void TestDocsAndFreqsAndPositionsAndPayloads()
+        {
+            base.TestDocsAndFreqsAndPositionsAndPayloads();
+        }
+
+        [Test]
+        public override void TestDocsAndFreqsAndPositionsAndOffsets()
+        {
+            base.TestDocsAndFreqsAndPositionsAndOffsets();
+        }
+
+        [Test]
+        public override void TestDocsAndFreqsAndPositionsAndOffsetsAndPayloads()
+        {
+            base.TestDocsAndFreqsAndPositionsAndOffsetsAndPayloads();
+        }
+
+        [Test]
+        public override void TestRandom()
+        {
+            base.TestRandom();
+        }
+
+        #endregion
+    }
+}
\ No newline at end of file


Mime
View raw message