lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [15/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests: Removed \core directory and put its contents in root directory
Date Sun, 26 Feb 2017 23:37:03 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestTransactions.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestTransactions.cs b/src/Lucene.Net.Tests/Index/TestTransactions.cs
new file mode 100644
index 0000000..627b21c
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestTransactions.cs
@@ -0,0 +1,336 @@
+using System;
+using System.Threading;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+    using Lucene.Net.Support;
+    using NUnit.Framework;
+    using System.IO;
+    using Directory = Lucene.Net.Store.Directory;
+    using Document = Documents.Document;
+    using English = Lucene.Net.Util.English;
+    using Field = Field;
+    using FieldType = FieldType;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+    using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+    using StringField = StringField;
+
+    [TestFixture]
+    public class TestTransactions : LuceneTestCase
+    {
+        private static volatile bool DoFail;
+
+        private class RandomFailure : MockDirectoryWrapper.Failure
+        {
+            private readonly TestTransactions OuterInstance;
+
+            public RandomFailure(TestTransactions outerInstance)
+            {
+                this.OuterInstance = outerInstance;
+            }
+
+            public override void Eval(MockDirectoryWrapper dir)
+            {
+                if (TestTransactions.DoFail && Random().Next() % 10 <= 3)
+                {
+                    throw new IOException("now failing randomly but on purpose");
+                }
+            }
+        }
+
+        private abstract class TimedThread : ThreadClass
+        {
+            internal volatile bool Failed;
+            internal static float RUN_TIME_MSEC = AtLeast(500);
+            internal TimedThread[] AllThreads;
+
+            public abstract void DoWork();
+
+            internal TimedThread(TimedThread[] threads)
+            {
+                this.AllThreads = threads;
+            }
+
+            public override void Run()
+            {
+                long stopTime = Environment.TickCount + (long)(RUN_TIME_MSEC);
+
+                try
+                {
+                    do
+                    {
+                        if (AnyErrors())
+                        {
+                            break;
+                        }
+                        DoWork();
+                    } while (Environment.TickCount < stopTime);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(Thread.CurrentThread + ": exc");
+                    Console.Error.WriteLine(e.StackTrace);
+                    Failed = true;
+                }
+            }
+
+            internal virtual bool AnyErrors()
+            {
+                for (int i = 0; i < AllThreads.Length; i++)
+                {
+                    if (AllThreads[i] != null && AllThreads[i].Failed)
+                    {
+                        return true;
+                    }
+                }
+                return false;
+            }
+        }
+
+        private class IndexerThread : TimedThread
+        {
+            private readonly TestTransactions OuterInstance;
+            private IConcurrentMergeScheduler _scheduler1;
+            private IConcurrentMergeScheduler _scheduler2;
+            internal Directory Dir1;
+            internal Directory Dir2;
+            internal object @lock;
+            internal int NextID;
+
+            public IndexerThread(TestTransactions outerInstance, object @lock, 
+                Directory dir1, Directory dir2,
+                IConcurrentMergeScheduler scheduler1, IConcurrentMergeScheduler scheduler2,

+                TimedThread[] threads)
+                : base(threads)
+            {
+                _scheduler1 = scheduler1;
+                _scheduler2 = scheduler2;
+                this.OuterInstance = outerInstance;
+                this.@lock = @lock;
+                this.Dir1 = dir1;
+                this.Dir2 = dir2;
+            }
+
+            public override void DoWork()
+            {
+                var config = OuterInstance.NewIndexWriterConfig(TEST_VERSION_CURRENT, new
MockAnalyzer(Random()))
+                                .SetMaxBufferedDocs(3)
+                                .SetMergeScheduler(_scheduler1)
+                                .SetMergePolicy(NewLogMergePolicy(2));
+                IndexWriter writer1 = new IndexWriter(Dir1, config);
+                ((IConcurrentMergeScheduler)writer1.Config.MergeScheduler).SetSuppressExceptions();
+
+                // Intentionally use different params so flush/merge
+                // happen @ different times
+                var config2 = OuterInstance.NewIndexWriterConfig(TEST_VERSION_CURRENT, new
MockAnalyzer(Random()))
+                                .SetMaxBufferedDocs(2)
+                                .SetMergeScheduler(_scheduler2)
+                                .SetMergePolicy(NewLogMergePolicy(3));
+                IndexWriter writer2 = new IndexWriter(Dir2, config2);
+                ((IConcurrentMergeScheduler)writer2.Config.MergeScheduler).SetSuppressExceptions();
+
+                Update(writer1);
+                Update(writer2);
+
+                DoFail = true;
+                try
+                {
+                    lock (@lock)
+                    {
+                        try
+                        {
+                            writer1.PrepareCommit();
+                        }
+                        catch (Exception)
+                        {
+                            writer1.Rollback();
+                            writer2.Rollback();
+                            return;
+                        }
+                        try
+                        {
+                            writer2.PrepareCommit();
+                        }
+                        catch (Exception)
+                        {
+                            writer1.Rollback();
+                            writer2.Rollback();
+                            return;
+                        }
+
+                        writer1.Commit();
+                        writer2.Commit();
+                    }
+                }
+                finally
+                {
+                    DoFail = false;
+                }
+
+                writer1.Dispose();
+                writer2.Dispose();
+            }
+
+            public virtual void Update(IndexWriter writer)
+            {
+                // Add 10 docs:
+                FieldType customType = new FieldType(StringField.TYPE_NOT_STORED);
+                customType.StoreTermVectors = true;
+                for (int j = 0; j < 10; j++)
+                {
+                    Document d = new Document();
+                    int n = Random().Next();
+                    d.Add(OuterInstance.NewField("id", Convert.ToString(NextID++), customType));
+                    d.Add(OuterInstance.NewTextField("contents", English.IntToEnglish(n),
Field.Store.NO));
+                    writer.AddDocument(d);
+                }
+
+                // Delete 5 docs:
+                int deleteID = NextID - 1;
+                for (int j = 0; j < 5; j++)
+                {
+                    writer.DeleteDocuments(new Term("id", "" + deleteID));
+                    deleteID -= 2;
+                }
+            }
+        }
+
+        private class SearcherThread : TimedThread
+        {
+            internal Directory Dir1;
+            internal Directory Dir2;
+            internal object @lock;
+
+            public SearcherThread(object @lock, Directory dir1, Directory dir2, TimedThread[]
threads)
+                : base(threads)
+            {
+                this.@lock = @lock;
+                this.Dir1 = dir1;
+                this.Dir2 = dir2;
+            }
+
+            public override void DoWork()
+            {
+                IndexReader r1 = null, r2 = null;
+                lock (@lock)
+                {
+                    try
+                    {
+                        r1 = DirectoryReader.Open(Dir1);
+                        r2 = DirectoryReader.Open(Dir2);
+                    }
+                    catch (IOException e)
+                    {
+                        if (!e.Message.Contains("on purpose"))
+                        {
+                            throw e;
+                        }
+                        if (r1 != null)
+                        {
+                            r1.Dispose();
+                        }
+                        if (r2 != null)
+                        {
+                            r2.Dispose();
+                        }
+                        return;
+                    }
+                }
+                if (r1.NumDocs != r2.NumDocs)
+                {
+                    throw new Exception("doc counts differ: r1=" + r1.NumDocs + " r2=" +
r2.NumDocs);
+                }
+                r1.Dispose();
+                r2.Dispose();
+            }
+        }
+
+        public virtual void InitIndex(Directory dir)
+        {
+            IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(Random())));
+            for (int j = 0; j < 7; j++)
+            {
+                Document d = new Document();
+                int n = Random().Next();
+                d.Add(NewTextField("contents", English.IntToEnglish(n), Field.Store.NO));
+                writer.AddDocument(d);
+            }
+            writer.Dispose();
+        }
+
+        [Test, Sequential]
+        public virtual void TestTransactions_Mem(
+            [ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler
scheduler1, 
+            [ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler
scheduler2)
+        {
+            Console.WriteLine("Start test");
+            // we cant use non-ramdir on windows, because this test needs to double-write.
+            MockDirectoryWrapper dir1 = new MockDirectoryWrapper(Random(), new RAMDirectory());
+            MockDirectoryWrapper dir2 = new MockDirectoryWrapper(Random(), new RAMDirectory());
+            dir1.PreventDoubleWrite = false;
+            dir2.PreventDoubleWrite = false;
+            dir1.FailOn(new RandomFailure(this));
+            dir2.FailOn(new RandomFailure(this));
+            dir1.FailOnOpenInput = false;
+            dir2.FailOnOpenInput = false;
+
+            // We throw exceptions in deleteFile, which creates
+            // leftover files:
+            dir1.AssertNoUnrefencedFilesOnClose = false;
+            dir2.AssertNoUnrefencedFilesOnClose = false;
+
+            InitIndex(dir1);
+            InitIndex(dir2);
+
+            TimedThread[] threads = new TimedThread[3];
+            int numThread = 0;
+
+            IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, scheduler1,
scheduler2, threads);
+            threads[numThread++] = indexerThread;
+            indexerThread.Start();
+
+            SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads);
+            threads[numThread++] = searcherThread1;
+            searcherThread1.Start();
+
+            SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads);
+            threads[numThread++] = searcherThread2;
+            searcherThread2.Start();
+
+            for (int i = 0; i < numThread; i++)
+            {
+                threads[i].Join();
+            }
+
+            for (int i = 0; i < numThread; i++)
+            {
+                Assert.IsTrue(!threads[i].Failed);
+            }
+            dir1.Dispose();
+            dir2.Dispose();
+
+            Console.WriteLine("End test");
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestTryDelete.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestTryDelete.cs b/src/Lucene.Net.Tests/Index/TestTryDelete.cs
new file mode 100644
index 0000000..a865675
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestTryDelete.cs
@@ -0,0 +1,196 @@
+using System;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+    using Lucene.Net.Randomized.Generators;
+    using Lucene.Net.Search;
+    using NUnit.Framework;
+    using Directory = Lucene.Net.Store.Directory;
+    using Document = Documents.Document;
+    using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+    using ReferenceManager = Lucene.Net.Search.ReferenceManager;
+    using SearcherFactory = Lucene.Net.Search.SearcherFactory;
+    using SearcherManager = Lucene.Net.Search.SearcherManager;
+    using Store = Field.Store;
+    using StringField = StringField;
+    using TermQuery = Lucene.Net.Search.TermQuery;
+    using TopDocs = Lucene.Net.Search.TopDocs;
+
+    [TestFixture]
+    public class TestTryDelete : LuceneTestCase
+    {
+        private static IndexWriter GetWriter(Directory directory)
+        {
+            MergePolicy policy = new LogByteSizeMergePolicy();
+            IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+            conf.SetMergePolicy(policy);
+            conf.SetOpenMode(OpenMode.CREATE_OR_APPEND);
+
+            IndexWriter writer = new IndexWriter(directory, conf);
+
+            return writer;
+        }
+
+        private static Directory CreateIndex()
+        {
+            Directory directory = new RAMDirectory();
+
+            IndexWriter writer = GetWriter(directory);
+
+            for (int i = 0; i < 10; i++)
+            {
+                Document doc = new Document();
+                doc.Add(new StringField("foo", Convert.ToString(i), Store.YES));
+                writer.AddDocument(doc);
+            }
+
+            writer.Commit();
+            writer.Dispose();
+
+            return directory;
+        }
+
+        [Test]
+        public virtual void TestTryDeleteDocument()
+        {
+            Directory directory = CreateIndex();
+
+            IndexWriter writer = GetWriter(directory);
+
+            ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true,
new SearcherFactory());
+
+            TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
+
+            IndexSearcher searcher = mgr.Acquire();
+
+            TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+            Assert.AreEqual(1, topDocs.TotalHits);
+
+            long result;
+            if (Random().NextBoolean())
+            {
+                IndexReader r = DirectoryReader.Open(writer, true);
+                result = mgrWriter.TryDeleteDocument(r, 0);
+                r.Dispose();
+            }
+            else
+            {
+                result = mgrWriter.TryDeleteDocument(searcher.IndexReader, 0);
+            }
+
+            // The tryDeleteDocument should have succeeded:
+            Assert.IsTrue(result != -1);
+
+            Assert.IsTrue(writer.HasDeletions);
+
+            if (Random().NextBoolean())
+            {
+                writer.Commit();
+            }
+
+            Assert.IsTrue(writer.HasDeletions);
+
+            mgr.MaybeRefresh();
+
+            searcher = mgr.Acquire();
+
+            topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+
+            Assert.AreEqual(0, topDocs.TotalHits);
+        }
+
+        [Test]
+        public virtual void TestTryDeleteDocumentCloseAndReopen()
+        {
+            Directory directory = CreateIndex();
+
+            IndexWriter writer = GetWriter(directory);
+
+            ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true,
new SearcherFactory());
+
+            IndexSearcher searcher = mgr.Acquire();
+
+            TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+            Assert.AreEqual(1, topDocs.TotalHits);
+
+            TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
+            long result = mgrWriter.TryDeleteDocument(DirectoryReader.Open(writer, true),
0);
+
+            Assert.AreEqual(1, result);
+
+            writer.Commit();
+
+            Assert.IsTrue(writer.HasDeletions);
+
+            mgr.MaybeRefresh();
+
+            searcher = mgr.Acquire();
+
+            topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+
+            Assert.AreEqual(0, topDocs.TotalHits);
+
+            writer.Dispose();
+
+            searcher = new IndexSearcher(DirectoryReader.Open(directory));
+
+            topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+
+            Assert.AreEqual(0, topDocs.TotalHits);
+        }
+
+        [Test]
+        public virtual void TestDeleteDocuments()
+        {
+            Directory directory = CreateIndex();
+
+            IndexWriter writer = GetWriter(directory);
+
+            ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true,
new SearcherFactory());
+
+            IndexSearcher searcher = mgr.Acquire();
+
+            TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+            Assert.AreEqual(1, topDocs.TotalHits);
+
+            TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
+            long result = mgrWriter.DeleteDocuments(new TermQuery(new Term("foo", "0")));
+
+            Assert.AreEqual(1, result);
+
+            // writer.Commit();
+
+            Assert.IsTrue(writer.HasDeletions);
+
+            mgr.MaybeRefresh();
+
+            searcher = mgr.Acquire();
+
+            topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+
+            Assert.AreEqual(0, topDocs.TotalHits);
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestTwoPhaseCommitTool.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestTwoPhaseCommitTool.cs b/src/Lucene.Net.Tests/Index/TestTwoPhaseCommitTool.cs
new file mode 100644
index 0000000..58c6f3a
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestTwoPhaseCommitTool.cs
@@ -0,0 +1,178 @@
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Index
+{
+    using Lucene.Net.Randomized.Generators;
+    using NUnit.Framework;
+    using System.IO;
+
+    /*
+                 * Licensed to the Apache Software Foundation (ASF) under one or more
+                 * contributor license agreements.  See the NOTICE file distributed with
+                 * this work for additional information regarding copyright ownership.
+                 * The ASF licenses this file to You under the Apache License, Version 2.0
+                 * (the "License"); you may not use this file except in compliance with
+                 * the License.  You may obtain a copy of the License at
+                 *
+                 *     http://www.apache.org/licenses/LICENSE-2.0
+                 *
+                 * Unless required by applicable law or agreed to in writing, software
+                 * distributed under the License is distributed on an "AS IS" BASIS,
+                 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+                 * See the License for the specific language governing permissions and
+                 * limitations under the License.
+                 */
+
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    [TestFixture]
+    public class TestTwoPhaseCommitTool : LuceneTestCase
+    {
+        private class TwoPhaseCommitImpl : ITwoPhaseCommit
+        {
+            internal static bool CommitCalled = false;
+            internal readonly bool FailOnPrepare;
+            internal readonly bool FailOnCommit;
+            internal readonly bool FailOnRollback;
+            internal bool RollbackCalled = false;
+            internal IDictionary<string, string> PrepareCommitData = null;
+            internal IDictionary<string, string> CommitData = null;
+
+            public TwoPhaseCommitImpl(bool failOnPrepare, bool failOnCommit, bool failOnRollback)
+            {
+                this.FailOnPrepare = failOnPrepare;
+                this.FailOnCommit = failOnCommit;
+                this.FailOnRollback = failOnRollback;
+            }
+
+            public void PrepareCommit()
+            {
+                PrepareCommit(null);
+            }
+
+            public virtual void PrepareCommit(IDictionary<string, string> commitData)
+            {
+                this.PrepareCommitData = commitData;
+                Assert.IsFalse(CommitCalled, "commit should not have been called before all
prepareCommit were");
+                if (FailOnPrepare)
+                {
+                    throw new IOException("failOnPrepare");
+                }
+            }
+
+            public void Commit()
+            {
+                Commit(null);
+            }
+
+            public virtual void Commit(IDictionary<string, string> commitData)
+            {
+                this.CommitData = commitData;
+                CommitCalled = true;
+                if (FailOnCommit)
+                {
+                    throw new Exception("failOnCommit");
+                }
+            }
+
+            public void Rollback()
+            {
+                RollbackCalled = true;
+                if (FailOnRollback)
+                {
+                    throw new Exception("failOnRollback");
+                }
+            }
+        }
+
+        [SetUp]
+        public override void SetUp()
+        {
+            base.SetUp();
+            TwoPhaseCommitImpl.CommitCalled = false; // reset count before every test
+        }
+
+        [Test]
+        public virtual void TestPrepareThenCommit()
+        {
+            // tests that prepareCommit() is called on all objects before commit()
+            TwoPhaseCommitImpl[] objects = new TwoPhaseCommitImpl[2];
+            for (int i = 0; i < objects.Length; i++)
+            {
+                objects[i] = new TwoPhaseCommitImpl(false, false, false);
+            }
+
+            // following call will fail if commit() is called before all prepare() were
+            TwoPhaseCommitTool.Execute(objects);
+        }
+
+        [Test]
+        public virtual void TestRollback()
+        {
+            // tests that rollback is called if failure occurs at any stage
+            int numObjects = Random().Next(8) + 3; // between [3, 10]
+            TwoPhaseCommitImpl[] objects = new TwoPhaseCommitImpl[numObjects];
+            for (int i = 0; i < objects.Length; i++)
+            {
+                bool failOnPrepare = Random().NextBoolean();
+                // we should not hit failures on commit usually
+                bool failOnCommit = Random().NextDouble() < 0.05;
+                bool railOnRollback = Random().NextBoolean();
+                objects[i] = new TwoPhaseCommitImpl(failOnPrepare, failOnCommit, railOnRollback);
+            }
+
+            bool anyFailure = false;
+            try
+            {
+                TwoPhaseCommitTool.Execute(objects);
+            }
+#pragma warning disable 168
+            catch (Exception t)
+#pragma warning restore 168
+            {
+                anyFailure = true;
+            }
+
+            if (anyFailure)
+            {
+                // if any failure happened, ensure that rollback was called on all.
+                foreach (TwoPhaseCommitImpl tpc in objects)
+                {
+                    Assert.IsTrue(tpc.RollbackCalled, "rollback was not called while a failure
occurred during the 2-phase commit");
+                }
+            }
+        }
+
+        [Test]
+        public virtual void TestNullTPCs()
+        {
+            int numObjects = Random().Next(4) + 3; // between [3, 6]
+            ITwoPhaseCommit[] tpcs = new ITwoPhaseCommit[numObjects];
+            bool setNull = false;
+            for (int i = 0; i < tpcs.Length; i++)
+            {
+                bool isNull = Random().NextDouble() < 0.3;
+                if (isNull)
+                {
+                    setNull = true;
+                    tpcs[i] = null;
+                }
+                else
+                {
+                    tpcs[i] = new TwoPhaseCommitImpl(false, false, false);
+                }
+            }
+
+            if (!setNull)
+            {
+                // none of the TPCs were picked to be null, pick one at random
+                int idx = Random().Next(numObjects);
+                tpcs[idx] = null;
+            }
+
+            // following call would fail if TPCTool won't handle null TPCs properly
+            TwoPhaseCommitTool.Execute(tpcs);
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestUniqueTermCount.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestUniqueTermCount.cs b/src/Lucene.Net.Tests/Index/TestUniqueTermCount.cs
new file mode 100644
index 0000000..46f4722
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestUniqueTermCount.cs
@@ -0,0 +1,122 @@
+´╗┐using Lucene.Net.Analysis;
+using Lucene.Net.Documents;
+using Lucene.Net.Search;
+using Lucene.Net.Search.Similarities;
+using Lucene.Net.Store;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Lucene.Net.Index
+{
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+
+    /// <summary>
+    /// Tests the uniqueTermCount statistic in FieldInvertState
+    /// </summary>
+    public class TestUniqueTermCount : LuceneTestCase
+    {
+        Directory dir;
+        IndexReader reader;
+        /* expected uniqueTermCount values for our documents */
+        List<int> expected = new List<int>();
+
+        public override void SetUp()
+        {
+            base.SetUp();
+            dir = NewDirectory();
+            MockAnalyzer analyzer = new MockAnalyzer(Random(), MockTokenizer.SIMPLE, true);
+            IndexWriterConfig config = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+            config.SetMergePolicy(NewLogMergePolicy());
+            config.SetSimilarity(new TestSimilarity());
+            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, config);
+            Document doc = new Document();
+            Field foo = NewTextField("foo", "", Field.Store.NO);
+            doc.Add(foo);
+            for (int i = 0; i < 100; i++)
+            {
+                foo.SetStringValue(AddValue());
+                writer.AddDocument(doc);
+            }
+            reader = writer.Reader;
+            writer.Dispose();
+        }
+
+        public override void TearDown()
+        {
+            reader.Dispose();
+            dir.Dispose();
+            base.TearDown();
+        }
+
+        [Test]
+        public void Test()
+        {
+            NumericDocValues fooNorms = MultiDocValues.GetNormValues(reader, "foo");
+            assertNotNull(fooNorms);
+            for (int i = 0; i < reader.MaxDoc; i++)
+            {
+                assertEquals(expected[i], fooNorms.Get(i));
+            }
+        }
+
+        /**
+         * Makes a bunch of single-char tokens (the max # unique terms will at most be 26).
+         * puts the # unique terms into expected, to be checked against the norm.
+         */
+        private string AddValue()
+        {
+            StringBuilder sb = new StringBuilder();
+            HashSet<string> terms = new HashSet<string>();
+            int num = TestUtil.NextInt(Random(), 0, 255);
+            for (int i = 0; i < num; i++)
+            {
+                sb.append(' ');
+                char term = (char)TestUtil.NextInt(Random(), 'a', 'z');
+                sb.append(term);
+                terms.add("" + term);
+            }
+            expected.Add(terms.size());
+            return sb.toString();
+        }
+
+        /**
+         * Simple similarity that encodes maxTermFrequency directly
+         */
+        internal class TestSimilarity : Similarity
+        {
+
+            public override long ComputeNorm(FieldInvertState state)
+            {
+                return state.UniqueTermCount;
+            }
+
+            public override SimWeight ComputeWeight(float queryBoost, CollectionStatistics
collectionStats, params TermStatistics[] termStats)
+            {
+                throw new InvalidOperationException();
+            }
+
+            public override SimScorer GetSimScorer(SimWeight weight, AtomicReaderContext
context)
+            {
+                throw new InvalidOperationException();
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/bogus24.upgraded.to.36.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/bogus24.upgraded.to.36.zip b/src/Lucene.Net.Tests/Index/bogus24.upgraded.to.36.zip
new file mode 100644
index 0000000..52a09f9
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/bogus24.upgraded.to.36.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.30.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.30.cfs.zip b/src/Lucene.Net.Tests/Index/index.30.cfs.zip
new file mode 100644
index 0000000..d5978c8
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.30.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.30.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.30.nocfs.zip b/src/Lucene.Net.Tests/Index/index.30.nocfs.zip
new file mode 100644
index 0000000..28cd83b
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.30.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.31.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.31.cfs.zip b/src/Lucene.Net.Tests/Index/index.31.cfs.zip
new file mode 100644
index 0000000..8f123a7
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.31.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.31.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.31.nocfs.zip b/src/Lucene.Net.Tests/Index/index.31.nocfs.zip
new file mode 100644
index 0000000..21434e1
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.31.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.31.optimized.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.31.optimized.cfs.zip b/src/Lucene.Net.Tests/Index/index.31.optimized.cfs.zip
new file mode 100644
index 0000000..200c710
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.31.optimized.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.31.optimized.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.31.optimized.nocfs.zip b/src/Lucene.Net.Tests/Index/index.31.optimized.nocfs.zip
new file mode 100644
index 0000000..9a158f1
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.31.optimized.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.32.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.32.cfs.zip b/src/Lucene.Net.Tests/Index/index.32.cfs.zip
new file mode 100644
index 0000000..eff3153
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.32.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.32.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.32.nocfs.zip b/src/Lucene.Net.Tests/Index/index.32.nocfs.zip
new file mode 100644
index 0000000..0b345da
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.32.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.34.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.34.cfs.zip b/src/Lucene.Net.Tests/Index/index.34.cfs.zip
new file mode 100644
index 0000000..257e9d8
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.34.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.34.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.34.nocfs.zip b/src/Lucene.Net.Tests/Index/index.34.nocfs.zip
new file mode 100644
index 0000000..935d6a1
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.34.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.36.surrogates.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.36.surrogates.zip b/src/Lucene.Net.Tests/Index/index.36.surrogates.zip
new file mode 100644
index 0000000..6bd7f20
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.36.surrogates.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.40.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.40.cfs.zip b/src/Lucene.Net.Tests/Index/index.40.cfs.zip
new file mode 100644
index 0000000..4974749
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.40.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.40.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.40.nocfs.zip b/src/Lucene.Net.Tests/Index/index.40.nocfs.zip
new file mode 100644
index 0000000..9699080
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.40.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.40.optimized.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.40.optimized.cfs.zip b/src/Lucene.Net.Tests/Index/index.40.optimized.cfs.zip
new file mode 100644
index 0000000..209c436
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.40.optimized.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.40.optimized.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.40.optimized.nocfs.zip b/src/Lucene.Net.Tests/Index/index.40.optimized.nocfs.zip
new file mode 100644
index 0000000..0eaffd0
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.40.optimized.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.41.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.41.cfs.zip b/src/Lucene.Net.Tests/Index/index.41.cfs.zip
new file mode 100644
index 0000000..da2745e
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.41.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.41.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.41.nocfs.zip b/src/Lucene.Net.Tests/Index/index.41.nocfs.zip
new file mode 100644
index 0000000..c056bcb
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.41.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.42.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.42.cfs.zip b/src/Lucene.Net.Tests/Index/index.42.cfs.zip
new file mode 100644
index 0000000..5945fe5
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.42.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.42.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.42.nocfs.zip b/src/Lucene.Net.Tests/Index/index.42.nocfs.zip
new file mode 100644
index 0000000..11de1f1
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.42.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.45.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.45.cfs.zip b/src/Lucene.Net.Tests/Index/index.45.cfs.zip
new file mode 100644
index 0000000..10a8a1a
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.45.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.45.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.45.nocfs.zip b/src/Lucene.Net.Tests/Index/index.45.nocfs.zip
new file mode 100644
index 0000000..7825e2a
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.45.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.461.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.461.cfs.zip b/src/Lucene.Net.Tests/Index/index.461.cfs.zip
new file mode 100644
index 0000000..8f18185
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.461.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.461.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.461.nocfs.zip b/src/Lucene.Net.Tests/Index/index.461.nocfs.zip
new file mode 100644
index 0000000..cf0173c
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.461.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/moreterms.40.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/moreterms.40.zip b/src/Lucene.Net.Tests/Index/moreterms.40.zip
new file mode 100644
index 0000000..53ad7ce
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/moreterms.40.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.19.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.19.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.19.cfs.zip
new file mode 100644
index 0000000..4fd9b32
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.19.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.19.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.19.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.19.nocfs.zip
new file mode 100644
index 0000000..e0d9142
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.19.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.20.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.20.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.20.cfs.zip
new file mode 100644
index 0000000..4b931ae
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.20.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.20.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.20.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.20.nocfs.zip
new file mode 100644
index 0000000..1275cdf
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.20.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.21.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.21.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.21.cfs.zip
new file mode 100644
index 0000000..473c138
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.21.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.21.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.21.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.21.nocfs.zip
new file mode 100644
index 0000000..d0582d0
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.21.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.22.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.22.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.22.cfs.zip
new file mode 100644
index 0000000..1236307
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.22.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.22.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.22.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.22.nocfs.zip
new file mode 100644
index 0000000..216ddf3
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.22.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.23.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.23.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.23.cfs.zip
new file mode 100644
index 0000000..b5fdeef
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.23.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.23.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.23.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.23.nocfs.zip
new file mode 100644
index 0000000..9137ae6
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.23.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.24.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.24.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.24.cfs.zip
new file mode 100644
index 0000000..2c666a9
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.24.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.24.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.24.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.24.nocfs.zip
new file mode 100644
index 0000000..c223875
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.24.nocfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.29.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.29.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.29.cfs.zip
new file mode 100644
index 0000000..c694c78
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.29.cfs.zip differ

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.29.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.29.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.29.nocfs.zip
new file mode 100644
index 0000000..298cab7
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.29.nocfs.zip differ


Mime
View raw message