lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nightowl...@apache.org
Subject [34/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests: Removed \core directory and put its contents in root directory
Date Sun, 26 Feb 2017 23:37:22 GMT
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestDocumentsWriterDeleteQueue.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestDocumentsWriterDeleteQueue.cs b/src/Lucene.Net.Tests/Index/TestDocumentsWriterDeleteQueue.cs
new file mode 100644
index 0000000..b8de01b
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestDocumentsWriterDeleteQueue.cs
@@ -0,0 +1,302 @@
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using NUnit.Framework;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Threading;
+
+namespace Lucene.Net.Index
+{
+    using BytesRef = Lucene.Net.Util.BytesRef;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements. See the NOTICE file distributed with this
+         * work for additional information regarding copyright ownership. The ASF
+         * licenses this file to You under the Apache License, Version 2.0 (the
+         * "License"); you may not use this file except in compliance with the License.
+         * You may obtain a copy of the License at
+         *
+         * http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+         * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+         * License for the specific language governing permissions and limitations under
+         * the License.
+         */
+
+    using DeleteSlice = Lucene.Net.Index.DocumentsWriterDeleteQueue.DeleteSlice;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+    using TermQuery = Lucene.Net.Search.TermQuery;
+
+    /// <summary>
+    /// Unit test for <seealso cref="DocumentsWriterDeleteQueue"/>
+    /// </summary>
+    [TestFixture]
+    public class TestDocumentsWriterDeleteQueue : LuceneTestCase
+    {
+        [Test]
+        public virtual void TestUpdateDelteSlices()
+        {
+            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
+            int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
+            int?[] ids = new int?[size];
+            for (int i = 0; i < ids.Length; i++)
+            {
+                ids[i] = Random().Next();
+            }
+            DeleteSlice slice1 = queue.NewSlice();
+            DeleteSlice slice2 = queue.NewSlice();
+            BufferedUpdates bd1 = new BufferedUpdates();
+            BufferedUpdates bd2 = new BufferedUpdates();
+            int last1 = 0;
+            int last2 = 0;
+            HashSet<Term> uniqueValues = new HashSet<Term>();
+            for (int j = 0; j < ids.Length; j++)
+            {
+                int? i = ids[j];
+                // create an array here since we compare identity below against tailItem
+                Term[] term = new Term[] { new Term("id", i.ToString()) };
+                uniqueValues.Add(term[0]);
+                queue.AddDelete(term);
+                if (Random().Next(20) == 0 || j == ids.Length - 1)
+                {
+                    queue.UpdateSlice(slice1);
+                    Assert.IsTrue(slice1.IsTailItem(term));
+                    slice1.Apply(bd1, j);
+                    AssertAllBetween(last1, j, bd1, ids);
+                    last1 = j + 1;
+                }
+                if (Random().Next(10) == 5 || j == ids.Length - 1)
+                {
+                    queue.UpdateSlice(slice2);
+                    Assert.IsTrue(slice2.IsTailItem(term));
+                    slice2.Apply(bd2, j);
+                    AssertAllBetween(last2, j, bd2, ids);
+                    last2 = j + 1;
+                }
+                Assert.AreEqual(j + 1, queue.NumGlobalTermDeletes);
+            }
+            assertEquals(uniqueValues, new HashSet<Term>(bd1.terms.Keys));
+            assertEquals(uniqueValues, new HashSet<Term>(bd2.terms.Keys));
+            var frozenSet = new HashSet<Term>();
+            foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
+            {
+                BytesRef bytesRef = new BytesRef();
+                bytesRef.CopyBytes(t.Bytes);
+                frozenSet.Add(new Term(t.Field, bytesRef));
+            }
+            assertEquals(uniqueValues, frozenSet);
+            Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
+        }
+
+        private void AssertAllBetween(int start, int end, BufferedUpdates deletes, int?[] ids)
+        {
+            for (int i = start; i <= end; i++)
+            {
+                Assert.AreEqual(Convert.ToInt32(end), deletes.terms[new Term("id", ids[i].ToString())]);
+            }
+        }
+
+        [Test]
+        public virtual void TestClear()
+        {
+            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
+            Assert.IsFalse(queue.AnyChanges());
+            queue.Clear();
+            Assert.IsFalse(queue.AnyChanges());
+            int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
+            int termsSinceFreeze = 0;
+            int queriesSinceFreeze = 0;
+            for (int i = 0; i < size; i++)
+            {
+                Term term = new Term("id", "" + i);
+                if (Random().Next(10) == 0)
+                {
+                    queue.AddDelete(new TermQuery(term));
+                    queriesSinceFreeze++;
+                }
+                else
+                {
+                    queue.AddDelete(term);
+                    termsSinceFreeze++;
+                }
+                Assert.IsTrue(queue.AnyChanges());
+                if (Random().Next(10) == 0)
+                {
+                    queue.Clear();
+                    queue.TryApplyGlobalSlice();
+                    Assert.IsFalse(queue.AnyChanges());
+                }
+            }
+        }
+
+        [Test]
+        public virtual void TestAnyChanges()
+        {
+            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
+            int size = 200 + Random().Next(500) * RANDOM_MULTIPLIER;
+            int termsSinceFreeze = 0;
+            int queriesSinceFreeze = 0;
+            for (int i = 0; i < size; i++)
+            {
+                Term term = new Term("id", "" + i);
+                if (Random().Next(10) == 0)
+                {
+                    queue.AddDelete(new TermQuery(term));
+                    queriesSinceFreeze++;
+                }
+                else
+                {
+                    queue.AddDelete(term);
+                    termsSinceFreeze++;
+                }
+                Assert.IsTrue(queue.AnyChanges());
+                if (Random().Next(5) == 0)
+                {
+                    FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
+                    Assert.AreEqual(termsSinceFreeze, freezeGlobalBuffer.termCount);
+                    Assert.AreEqual(queriesSinceFreeze, ((Query[])freezeGlobalBuffer.queries.Clone()).Length);
+                    queriesSinceFreeze = 0;
+                    termsSinceFreeze = 0;
+                    Assert.IsFalse(queue.AnyChanges());
+                }
+            }
+        }
+
+        [Test]
+        public virtual void TestPartiallyAppliedGlobalSlice()
+        {
+            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
+            System.Reflection.FieldInfo field = typeof(DocumentsWriterDeleteQueue).GetField("globalBufferLock", 
+                BindingFlags.NonPublic | BindingFlags.GetField | BindingFlags.Instance);
+            ReentrantLock @lock = (ReentrantLock)field.GetValue(queue);
+            @lock.Lock();
+            ThreadClass t = new ThreadAnonymousInnerClassHelper(this, queue);
+            t.Start();
+            t.Join();
+            @lock.Unlock();
+            Assert.IsTrue(queue.AnyChanges(), "changes in del queue but not in slice yet");
+            queue.TryApplyGlobalSlice();
+            Assert.IsTrue(queue.AnyChanges(), "changes in global buffer");
+            FrozenBufferedUpdates freezeGlobalBuffer = queue.FreezeGlobalBuffer(null);
+            Assert.IsTrue(freezeGlobalBuffer.Any());
+            Assert.AreEqual(1, freezeGlobalBuffer.termCount);
+            Assert.IsFalse(queue.AnyChanges(), "all changes applied");
+        }
+
+        private class ThreadAnonymousInnerClassHelper : ThreadClass
+        {
+            private readonly TestDocumentsWriterDeleteQueue OuterInstance;
+
+            private DocumentsWriterDeleteQueue Queue;
+
+            public ThreadAnonymousInnerClassHelper(TestDocumentsWriterDeleteQueue outerInstance, DocumentsWriterDeleteQueue queue)
+            {
+                this.OuterInstance = outerInstance;
+                this.Queue = queue;
+            }
+
+            public override void Run()
+            {
+                Queue.AddDelete(new Term("foo", "bar"));
+            }
+        }
+
+        [Test]
+        public virtual void TestStressDeleteQueue()
+        {
+            DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
+            HashSet<Term> uniqueValues = new HashSet<Term>();
+            int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER;
+            int?[] ids = new int?[size];
+            for (int i = 0; i < ids.Length; i++)
+            {
+                ids[i] = Random().Next();
+                uniqueValues.Add(new Term("id", ids[i].ToString()));
+            }
+            CountdownEvent latch = new CountdownEvent(1);
+            AtomicInt32 index = new AtomicInt32(0);
+            int numThreads = 2 + Random().Next(5);
+            UpdateThread[] threads = new UpdateThread[numThreads];
+            for (int i = 0; i < threads.Length; i++)
+            {
+                threads[i] = new UpdateThread(queue, index, ids, latch);
+                threads[i].Start();
+            }
+            latch.Signal();
+            for (int i = 0; i < threads.Length; i++)
+            {
+                threads[i].Join();
+            }
+
+            foreach (UpdateThread updateThread in threads)
+            {
+                DeleteSlice slice = updateThread.Slice;
+                queue.UpdateSlice(slice);
+                BufferedUpdates deletes = updateThread.Deletes;
+                slice.Apply(deletes, BufferedUpdates.MAX_INT32);
+                assertEquals(uniqueValues, new HashSet<Term>(deletes.terms.Keys));
+            }
+            queue.TryApplyGlobalSlice();
+            HashSet<Term> frozenSet = new HashSet<Term>();
+            foreach (Term t in queue.FreezeGlobalBuffer(null).GetTermsEnumerable())
+            {
+                BytesRef bytesRef = new BytesRef();
+                bytesRef.CopyBytes(t.Bytes);
+                frozenSet.Add(new Term(t.Field, bytesRef));
+            }
+            Assert.AreEqual(0, queue.NumGlobalTermDeletes, "num deletes must be 0 after freeze");
+            Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
+            assertEquals(uniqueValues, frozenSet);
+        }
+
+        private class UpdateThread : ThreadClass
+        {
+            internal readonly DocumentsWriterDeleteQueue Queue;
+            internal readonly AtomicInt32 Index;
+            internal readonly int?[] Ids;
+            internal readonly DeleteSlice Slice;
+            internal readonly BufferedUpdates Deletes;
+            internal readonly CountdownEvent Latch;
+
+            protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInt32 index, int?[] ids, CountdownEvent latch)
+            {
+                this.Queue = queue;
+                this.Index = index;
+                this.Ids = ids;
+                this.Slice = queue.NewSlice();
+                Deletes = new BufferedUpdates();
+                this.Latch = latch;
+            }
+
+            public override void Run()
+            {
+#if !NETSTANDARD
+                try
+                {
+#endif
+                    Latch.Wait();
+#if !NETSTANDARD
+                }
+                catch (ThreadInterruptedException e)
+                {
+                    throw new ThreadInterruptedException("Thread Interrupted Exception", e);
+                }
+#endif
+
+                int i = 0;
+                while ((i = Index.GetAndIncrement()) < Ids.Length)
+                {
+                    Term term = new Term("id", Ids[i].ToString());
+                    Queue.Add(term, Slice);
+                    Assert.IsTrue(Slice.IsTailItem(term));
+                    Slice.Apply(Deletes, BufferedUpdates.MAX_INT32);
+                }
+            }
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestDocumentsWriterStallControl.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestDocumentsWriterStallControl.cs b/src/Lucene.Net.Tests/Index/TestDocumentsWriterStallControl.cs
new file mode 100644
index 0000000..9459fca
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestDocumentsWriterStallControl.cs
@@ -0,0 +1,473 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+
+namespace Lucene.Net.Index
+{
+    using Lucene.Net.Randomized.Generators;
+    using Lucene.Net.Support;
+    using NUnit.Framework;
+    /*
+             * Licensed to the Apache Software Foundation (ASF) under one or more
+             * contributor license agreements. See the NOTICE file distributed with this
+             * work for additional information regarding copyright ownership. The ASF
+             * licenses this file to You under the Apache License, Version 2.0 (the
+             * "License"); you may not use this file except in compliance with the License.
+             * You may obtain a copy of the License at
+             *
+             * http://www.apache.org/licenses/LICENSE-2.0
+             *
+             * Unless required by applicable law or agreed to in writing, software
+             * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+             * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+             * License for the specific language governing permissions and limitations under
+             * the License.
+             */
+
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    /// <summary>
+    /// Tests for <seealso cref="DocumentsWriterStallControl"/>
+    /// </summary>
+    [TestFixture]
+    public class TestDocumentsWriterStallControl : LuceneTestCase
+    {
+        [Test]
+        public virtual void TestSimpleStall()
+        {
+            DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
+
+            ctrl.UpdateStalled(false);
+            ThreadClass[] waitThreads = WaitThreads(AtLeast(1), ctrl);
+            Start(waitThreads);
+            Assert.IsFalse(ctrl.HasBlocked);
+            Assert.IsFalse(ctrl.AnyStalledThreads());
+            Join(waitThreads);
+
+            // now stall threads and wake them up again
+            ctrl.UpdateStalled(true);
+            waitThreads = WaitThreads(AtLeast(1), ctrl);
+            Start(waitThreads);
+            AwaitState(ThreadState.WaitSleepJoin, waitThreads);
+            Assert.IsTrue(ctrl.HasBlocked);
+            Assert.IsTrue(ctrl.AnyStalledThreads());
+            ctrl.UpdateStalled(false);
+            Assert.IsFalse(ctrl.AnyStalledThreads());
+            Join(waitThreads);
+        }
+
+        [Test]
+        public virtual void TestRandom()
+        {
+            DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
+            ctrl.UpdateStalled(false);
+
+            ThreadClass[] stallThreads = new ThreadClass[AtLeast(3)];
+            for (int i = 0; i < stallThreads.Length; i++)
+            {
+                int stallProbability = 1 + Random().Next(10);
+                stallThreads[i] = new ThreadAnonymousInnerClassHelper(ctrl, stallProbability);
+            }
+            Start(stallThreads);
+            long time = Environment.TickCount;
+            /*
+             * use a 100 sec timeout to make sure we not hang forever. join will fail in
+             * that case
+             */
+            while ((Environment.TickCount - time) < 100 * 1000 && !Terminated(stallThreads))
+            {
+                ctrl.UpdateStalled(false);
+                if (Random().NextBoolean())
+                {
+                    Thread.Sleep(0);
+                }
+                else
+                {
+                    Thread.Sleep(1);
+                }
+            }
+            Join(stallThreads);
+        }
+
+        private class ThreadAnonymousInnerClassHelper : ThreadClass
+        {
+            private DocumentsWriterStallControl Ctrl;
+            private int StallProbability;
+
+            public ThreadAnonymousInnerClassHelper(DocumentsWriterStallControl ctrl, int stallProbability)
+            {
+                this.Ctrl = ctrl;
+                this.StallProbability = stallProbability;
+            }
+
+            public override void Run()
+            {
+                int iters = AtLeast(1000);
+                for (int j = 0; j < iters; j++)
+                {
+                    Ctrl.UpdateStalled(Random().Next(StallProbability) == 0);
+                    if (Random().Next(5) == 0) // thread 0 only updates
+                    {
+                        Ctrl.WaitIfStalled();
+                    }
+                }
+            }
+        }
+
+        [Test]
+        public virtual void TestAccquireReleaseRace()
+        {
+            DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
+            ctrl.UpdateStalled(false);
+            AtomicBoolean stop = new AtomicBoolean(false);
+            AtomicBoolean checkPoint = new AtomicBoolean(true);
+
+            int numStallers = AtLeast(1);
+            int numReleasers = AtLeast(1);
+            int numWaiters = AtLeast(1);
+            var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
+            var threads = new ThreadClass[numReleasers + numStallers + numWaiters];
+            IList<Exception> exceptions = new SynchronizedList<Exception>();
+            for (int i = 0; i < numReleasers; i++)
+            {
+                threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions);
+            }
+            for (int i = numReleasers; i < numReleasers + numStallers; i++)
+            {
+                threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions);
+            }
+            for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++)
+            {
+                threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions);
+            }
+
+            Start(threads);
+            int iters = AtLeast(10000);
+            float checkPointProbability = TEST_NIGHTLY ? 0.5f : 0.1f;
+            for (int i = 0; i < iters; i++)
+            {
+                if (checkPoint.Get())
+                {
+                    Assert.IsTrue(sync.UpdateJoin.Wait(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?");
+                    if (exceptions.Count > 0)
+                    {
+                        foreach (Exception throwable in exceptions)
+                        {
+                            Console.WriteLine(throwable.ToString());
+                            Console.Write(throwable.StackTrace);
+                        }
+                        Assert.Fail("got exceptions in threads");
+                    }
+
+                    if (ctrl.HasBlocked && ctrl.IsHealthy)
+                    {
+                        AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
+                    }
+
+                    checkPoint.Set(false);
+                    sync.Waiter.Signal();
+                    sync.LeftCheckpoint.Wait();
+                }
+                Assert.IsFalse(checkPoint.Get());
+                Assert.AreEqual(0, sync.Waiter.CurrentCount);
+                if (checkPointProbability >= (float)Random().NextDouble())
+                {
+                    sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
+                    checkPoint.Set(true);
+                }
+            }
+            if (!checkPoint.Get())
+            {
+                sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
+                checkPoint.Set(true);
+            }
+
+            Assert.IsTrue(sync.UpdateJoin.Wait(new TimeSpan(0, 0, 0, 10)));
+            AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
+            checkPoint.Set(false);
+            stop.Set(true);
+            sync.Waiter.Signal();
+            sync.LeftCheckpoint.Wait();
+
+            for (int i = 0; i < threads.Length; i++)
+            {
+                ctrl.UpdateStalled(false);
+                threads[i].Join(2000);
+                if (threads[i].IsAlive && threads[i] is Waiter)
+                {
+                    if (threads[i].State == ThreadState.WaitSleepJoin)
+                    {
+                        Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads());
+                    }
+                }
+            }
+        }
+
+        private void AssertState(int numReleasers, int numStallers, int numWaiters, ThreadClass[] threads, DocumentsWriterStallControl ctrl)
+        {
+            int millisToSleep = 100;
+            while (true)
+            {
+                if (ctrl.HasBlocked && ctrl.IsHealthy)
+                {
+                    for (int n = numReleasers + numStallers; n < numReleasers + numStallers + numWaiters; n++)
+                    {
+                        if (ctrl.IsThreadQueued(threads[n]))
+                        {
+                            if (millisToSleep < 60000)
+                            {
+                                Thread.Sleep(millisToSleep);
+                                millisToSleep *= 2;
+                                break;
+                            }
+                            else
+                            {
+                                Assert.Fail("control claims no stalled threads but waiter seems to be blocked ");
+                            }
+                        }
+                    }
+                    break;
+                }
+                else
+                {
+                    break;
+                }
+            }
+        }
+
+        internal class Waiter : ThreadClass
+        {
+            internal Synchronizer Sync;
+            internal DocumentsWriterStallControl Ctrl;
+            internal AtomicBoolean CheckPoint;
+            internal AtomicBoolean Stop;
+            internal IList<Exception> Exceptions;
+
+            public Waiter(AtomicBoolean stop, AtomicBoolean checkPoint, DocumentsWriterStallControl ctrl, Synchronizer sync, IList<Exception> exceptions)
+                : base("waiter")
+            {
+                this.Stop = stop;
+                this.CheckPoint = checkPoint;
+                this.Ctrl = ctrl;
+                this.Sync = sync;
+                this.Exceptions = exceptions;
+            }
+
+            public override void Run()
+            {
+                try
+                {
+                    while (!Stop.Get())
+                    {
+                        Ctrl.WaitIfStalled();
+                        if (CheckPoint.Get())
+                        {
+#if !NETSTANDARD
+                            try
+                            {
+#endif
+                                Assert.IsTrue(Sync.await());
+#if !NETSTANDARD
+                            }
+                            catch (ThreadInterruptedException e)
+                            {
+                                Console.WriteLine("[Waiter] got interrupted - wait count: " + Sync.Waiter.CurrentCount);
+                                throw new ThreadInterruptedException("Thread Interrupted Exception", e);
+                            }
+#endif
+                        }
+                    }
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e.ToString());
+                    Console.Write(e.StackTrace);
+                    Exceptions.Add(e);
+                }
+            }
+        }
+
+        internal class Updater : ThreadClass
+        {
+            internal Synchronizer Sync;
+            internal DocumentsWriterStallControl Ctrl;
+            internal AtomicBoolean CheckPoint;
+            internal AtomicBoolean Stop;
+            internal bool Release;
+            internal IList<Exception> Exceptions;
+
+            public Updater(AtomicBoolean stop, AtomicBoolean checkPoint, DocumentsWriterStallControl ctrl, Synchronizer sync, bool release, IList<Exception> exceptions)
+                : base("updater")
+            {
+                this.Stop = stop;
+                this.CheckPoint = checkPoint;
+                this.Ctrl = ctrl;
+                this.Sync = sync;
+                this.Release = release;
+                this.Exceptions = exceptions;
+            }
+
+            public override void Run()
+            {
+                try
+                {
+                    while (!Stop.Get())
+                    {
+                        int internalIters = Release && Random().NextBoolean() ? AtLeast(5) : 1;
+                        for (int i = 0; i < internalIters; i++)
+                        {
+                            Ctrl.UpdateStalled(Random().NextBoolean());
+                        }
+                        if (CheckPoint.Get())
+                        {
+                            Sync.UpdateJoin.Signal();
+                            try
+                            {
+                                Assert.IsTrue(Sync.await());
+                            }
+#if !NETSTANDARD
+                            catch (ThreadInterruptedException e)
+                            {
+                                Console.WriteLine("[Updater] got interrupted - wait count: " + Sync.Waiter.CurrentCount);
+                                throw new ThreadInterruptedException("Thread Interrupted Exception", e);
+                            }
+#endif
+                            catch (Exception e)
+                            {
+                                Console.Write("signal failed with : " + e);
+                                throw e;
+                            }
+
+                            Sync.LeftCheckpoint.Signal();
+                        }
+                        if (Random().NextBoolean())
+                        {
+                            Thread.Sleep(0);
+                        }
+                    }
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine(e.ToString());
+                    Console.Write(e.StackTrace);
+                    Exceptions.Add(e);
+                }
+
+                if (!Sync.UpdateJoin.IsSet)
+                {
+                    Sync.UpdateJoin.Signal();
+                }
+            }
+        }
+
+        public static bool Terminated(ThreadClass[] threads)
+        {
+            foreach (ThreadClass thread in threads)
+            {
+                if (ThreadState.Stopped != thread.State)
+                {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        public static void Start(ThreadClass[] tostart)
+        {
+            foreach (ThreadClass thread in tostart)
+            {
+                thread.Start();
+            }
+            Thread.Sleep(1); // let them start
+        }
+
+        public static void Join(ThreadClass[] toJoin)
+        {
+            foreach (ThreadClass thread in toJoin)
+            {
+                thread.Join();
+            }
+        }
+
+        internal static ThreadClass[] WaitThreads(int num, DocumentsWriterStallControl ctrl)
+        {
+            ThreadClass[] array = new ThreadClass[num];
+            for (int i = 0; i < array.Length; i++)
+            {
+                array[i] = new ThreadAnonymousInnerClassHelper2(ctrl);
+            }
+            return array;
+        }
+
+        private class ThreadAnonymousInnerClassHelper2 : ThreadClass
+        {
+            private DocumentsWriterStallControl Ctrl;
+
+            public ThreadAnonymousInnerClassHelper2(DocumentsWriterStallControl ctrl)
+            {
+                this.Ctrl = ctrl;
+            }
+
+            public override void Run()
+            {
+                Ctrl.WaitIfStalled();
+            }
+        }
+
+        /// <summary>
+        /// Waits for all incoming threads to be in wait()
+        ///  methods.
+        /// </summary>
+        public static void AwaitState(ThreadState state, params ThreadClass[] threads)
+        {
+            while (true)
+            {
+                bool done = true;
+                foreach (ThreadClass thread in threads)
+                {
+                    if (thread.State != state)
+                    {
+                        done = false;
+                        break;
+                    }
+                }
+                if (done)
+                {
+                    return;
+                }
+                if (Random().NextBoolean())
+                {
+                    Thread.Sleep(0);
+                }
+                else
+                {
+                    Thread.Sleep(1);
+                }
+            }
+        }
+
+        public sealed class Synchronizer
+        {
+            internal volatile CountdownEvent Waiter;
+            internal volatile CountdownEvent UpdateJoin;
+            internal volatile CountdownEvent LeftCheckpoint;
+
+            public Synchronizer(int numUpdater, int numThreads)
+            {
+                Reset(numUpdater, numThreads);
+            }
+
+            public void Reset(int numUpdaters, int numThreads)
+            {
+                this.Waiter = new CountdownEvent(1);
+                this.UpdateJoin = new CountdownEvent(numUpdaters);
+                this.LeftCheckpoint = new CountdownEvent(numUpdaters);
+            }
+
+            public bool @await()
+            {
+                return Waiter.Wait(new TimeSpan(0, 0, 0, 10));
+            }
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestDuelingCodecs.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestDuelingCodecs.cs b/src/Lucene.Net.Tests/Index/TestDuelingCodecs.cs
new file mode 100644
index 0000000..4ac4e65
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestDuelingCodecs.cs
@@ -0,0 +1,183 @@
+using Lucene.Net.Documents;
+using NUnit.Framework;
+using System;
+
+namespace Lucene.Net.Index
+{
+    using System.Text.RegularExpressions;
+    using Attributes;
+    using BytesRef = Lucene.Net.Util.BytesRef;
+    using Codec = Lucene.Net.Codecs.Codec;
+    using Directory = Lucene.Net.Store.Directory;
+    using Document = Documents.Document;
+    using LineFileDocs = Lucene.Net.Util.LineFileDocs;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using NumericDocValuesField = NumericDocValuesField;
+    using SortedSetDocValuesField = SortedSetDocValuesField;
+    using TestUtil = Lucene.Net.Util.TestUtil;
+
+    /// <summary>
+    /// Compares one codec against another
+    /// </summary>
+    [TestFixture]
+    public class TestDuelingCodecs : LuceneTestCase
+    {
+        private Directory LeftDir;
+        private IndexReader LeftReader;
+        private Codec LeftCodec;
+
+        private Directory RightDir;
+        private IndexReader RightReader;
+        private Codec RightCodec;
+
+        private string Info; // for debugging
+
+        [SetUp]
+        public override void SetUp()
+        {
+            base.SetUp();
+
+            // for now its SimpleText vs Lucene46(random postings format)
+            // as this gives the best overall coverage. when we have more
+            // codecs we should probably pick 2 from Codec.availableCodecs()
+
+            LeftCodec = Codec.ForName("SimpleText");
+            RightCodec = new RandomCodec(Random());
+
+            LeftDir = NewDirectory();
+            RightDir = NewDirectory();
+
+            long seed = Random().Next();
+
+            // must use same seed because of random payloads, etc
+            int maxTermLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
+            MockAnalyzer leftAnalyzer = new MockAnalyzer(new Random((int)seed));
+            leftAnalyzer.MaxTokenLength = maxTermLength;
+            MockAnalyzer rightAnalyzer = new MockAnalyzer(new Random((int)seed));
+            rightAnalyzer.MaxTokenLength = maxTermLength;
+
+            // but these can be different
+            // TODO: this turns this into a really big test of Multi*, is that what we want?
+            IndexWriterConfig leftConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, leftAnalyzer);
+            leftConfig.SetCodec(LeftCodec);
+            // preserve docids
+            leftConfig.SetMergePolicy(NewLogMergePolicy());
+
+            IndexWriterConfig rightConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, rightAnalyzer);
+            rightConfig.SetCodec(RightCodec);
+            // preserve docids
+            rightConfig.SetMergePolicy(NewLogMergePolicy());
+
+            // must use same seed because of random docvalues fields, etc
+            RandomIndexWriter leftWriter = new RandomIndexWriter(new Random((int)seed), LeftDir, leftConfig);
+            RandomIndexWriter rightWriter = new RandomIndexWriter(new Random((int)seed), RightDir, rightConfig);
+
+            int numdocs = AtLeast(100);
+            CreateRandomIndex(numdocs, leftWriter, seed);
+            CreateRandomIndex(numdocs, rightWriter, seed);
+
+            LeftReader = MaybeWrapReader(leftWriter.Reader);
+            leftWriter.Dispose();
+            RightReader = MaybeWrapReader(rightWriter.Reader);
+            rightWriter.Dispose();
+
+            // check that our readers are valid
+            TestUtil.CheckReader(LeftReader);
+            TestUtil.CheckReader(RightReader);
+
+            Info = "left: " + LeftCodec.ToString() + " / right: " + RightCodec.ToString();
+        }
+
+        [TearDown]
+        public override void TearDown()
+        {
+            if (LeftReader != null)
+            {
+                LeftReader.Dispose();
+            }
+            if (RightReader != null)
+            {
+                RightReader.Dispose();
+            }
+
+            if (LeftDir != null)
+            {
+                LeftDir.Dispose();
+            }
+            if (RightDir != null)
+            {
+                RightDir.Dispose();
+            }
+
+            base.TearDown();
+        }
+
+        /// <summary>
+        /// populates a writer with random stuff. this must be fully reproducable with the seed!
+        /// </summary>
+        public static void CreateRandomIndex(int numdocs, RandomIndexWriter writer, long seed)
+        {
+            Random random = new Random((int)seed);
+            // primary source for our data is from linefiledocs, its realistic.
+            LineFileDocs lineFileDocs = new LineFileDocs(random);
+
+            // LUCENENET: compile a regex so we don't have to do it in each loop (for regex.split())
+            Regex whiteSpace = new Regex("\\s+", RegexOptions.Compiled);
+
+            // TODO: we should add other fields that use things like docs&freqs but omit positions,
+            // because linefiledocs doesn't cover all the possibilities.
+            for (int i = 0; i < numdocs; i++)
+            {
+                Document document = lineFileDocs.NextDoc();
+                // grab the title and add some SortedSet instances for fun
+                string title = document.Get("titleTokenized");
+                string[] split = whiteSpace.Split(title);
+                foreach (string trash in split)
+                {
+                    document.Add(new SortedSetDocValuesField("sortedset", new BytesRef(trash)));
+                }
+                // add a numeric dv field sometimes
+                document.RemoveFields("sparsenumeric");
+                if (random.Next(4) == 2)
+                {
+                    document.Add(new NumericDocValuesField("sparsenumeric", random.Next()));
+                }
+                writer.AddDocument(document);
+            }
+
+            lineFileDocs.Dispose();
+        }
+
+        /// <summary>
+        /// checks the two indexes are equivalent
+        /// </summary>
+#if !NETSTANDARD
+        // LUCENENET: There is no Timeout on NUnit for .NET Core.
+        [Timeout(120000)]
+#endif
+        [Test, HasTimeout]
+        public virtual void TestEquals()
+        {
+            AssertReaderEquals(Info, LeftReader, RightReader);
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestExceedMaxTermLength.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestExceedMaxTermLength.cs b/src/Lucene.Net.Tests/Index/TestExceedMaxTermLength.cs
new file mode 100644
index 0000000..aa3470d
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestExceedMaxTermLength.cs
@@ -0,0 +1,108 @@
+using System;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+    using Lucene.Net.Randomized.Generators;
+    using NUnit.Framework;
+    using Directory = Lucene.Net.Store.Directory;
+    using Document = Documents.Document;
+    using Field = Field;
+    using FieldType = FieldType;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using TestUtil = Lucene.Net.Util.TestUtil;
+
+    /// <summary>
+    /// Tests that a useful exception is thrown when attempting to index a term that is
+    /// too large
+    /// </summary>
+    /// <seealso cref= IndexWriter#MAX_TERM_LENGTH </seealso>
+    [TestFixture]
+    public class TestExceedMaxTermLength : LuceneTestCase
+    {
+        private static readonly int MinTestTermLength = IndexWriter.MAX_TERM_LENGTH + 1;
+        private static readonly int MaxTestTermLegnth = IndexWriter.MAX_TERM_LENGTH * 2;
+
+        internal Directory Dir = null;
+
+        [SetUp]
+        public virtual void CreateDir()
+        {
+            Dir = NewDirectory();
+        }
+
+        [TearDown]
+        public virtual void DestroyDir()
+        {
+            Dir.Dispose();
+            Dir = null;
+        }
+
+        [Test]
+        public virtual void Test()
+        {
+            IndexWriter w = new IndexWriter(Dir, NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+            try
+            {
+                FieldType ft = new FieldType();
+                ft.IsIndexed = true;
+                ft.IsStored = Random().NextBoolean();
+                ft.Freeze();
+
+                Document doc = new Document();
+                if (Random().NextBoolean())
+                {
+                    // totally ok short field value
+                    doc.Add(new Field(TestUtil.RandomSimpleString(Random(), 1, 10), TestUtil.RandomSimpleString(Random(), 1, 10), ft));
+                }
+                // problematic field
+                string name = TestUtil.RandomSimpleString(Random(), 1, 50);
+                string value = TestUtil.RandomSimpleString(Random(), MinTestTermLength, MaxTestTermLegnth);
+                Field f = new Field(name, value, ft);
+                if (Random().NextBoolean())
+                {
+                    // totally ok short field value
+                    doc.Add(new Field(TestUtil.RandomSimpleString(Random(), 1, 10), TestUtil.RandomSimpleString(Random(), 1, 10), ft));
+                }
+                doc.Add(f);
+
+                try
+                {
+                    w.AddDocument(doc);
+                    Assert.Fail("Did not get an exception from adding a monster term");
+                }
+                catch (System.ArgumentException e)
+                {
+                    string maxLengthMsg = Convert.ToString(IndexWriter.MAX_TERM_LENGTH);
+                    string msg = e.Message;
+                    Assert.IsTrue(msg.Contains("immense term"), "IllegalArgumentException didn't mention 'immense term': " + msg);
+                    Assert.IsTrue(msg.Contains(maxLengthMsg), "IllegalArgumentException didn't mention max length (" + maxLengthMsg + "): " + msg);
+                    Assert.IsTrue(msg.Contains(name), "IllegalArgumentException didn't mention field name (" + name + "): " + msg);
+                }
+            }
+            finally
+            {
+                w.Dispose();
+            }
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestFieldInfos.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestFieldInfos.cs b/src/Lucene.Net.Tests/Index/TestFieldInfos.cs
new file mode 100644
index 0000000..35f8d8a
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestFieldInfos.cs
@@ -0,0 +1,126 @@
+namespace Lucene.Net.Index
+{
+    using NUnit.Framework;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using Codec = Lucene.Net.Codecs.Codec;
+    using Directory = Lucene.Net.Store.Directory;
+    using Document = Documents.Document;
+    using FieldInfosReader = Lucene.Net.Codecs.FieldInfosReader;
+    using FieldInfosWriter = Lucene.Net.Codecs.FieldInfosWriter;
+    using IndexOutput = Lucene.Net.Store.IndexOutput;
+    using IOContext = Lucene.Net.Store.IOContext;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    //import org.cnlp.utils.properties.ResourceBundleHelper;
+
+    [TestFixture]
+    public class TestFieldInfos : LuceneTestCase
+    {
+        private Document TestDoc = new Document();
+
+        [SetUp]
+        public override void SetUp()
+        {
+            base.SetUp();
+            DocHelper.SetupDoc(TestDoc);
+        }
+
+        public virtual FieldInfos CreateAndWriteFieldInfos(Directory dir, string filename)
+        {
+            //Positive test of FieldInfos
+            Assert.IsTrue(TestDoc != null);
+            FieldInfos.Builder builder = new FieldInfos.Builder();
+            foreach (IIndexableField field in TestDoc)
+            {
+                builder.AddOrUpdate(field.Name, field.FieldType);
+            }
+            FieldInfos fieldInfos = builder.Finish();
+            //Since the complement is stored as well in the fields map
+            Assert.IsTrue(fieldInfos.Count == DocHelper.All.Count); //this is all b/c we are using the no-arg constructor
+
+            IndexOutput output = dir.CreateOutput(filename, NewIOContext(Random()));
+            Assert.IsTrue(output != null);
+            //Use a RAMOutputStream
+
+            FieldInfosWriter writer = Codec.Default.FieldInfosFormat.FieldInfosWriter;
+            writer.Write(dir, filename, "", fieldInfos, IOContext.DEFAULT);
+            output.Dispose();
+            return fieldInfos;
+        }
+
+        public virtual FieldInfos ReadFieldInfos(Directory dir, string filename)
+        {
+            FieldInfosReader reader = Codec.Default.FieldInfosFormat.FieldInfosReader;
+            return reader.Read(dir, filename, "", IOContext.DEFAULT);
+        }
+
+        [Test]
+        public virtual void Test()
+        {
+            string name = "testFile";
+            Directory dir = NewDirectory();
+            FieldInfos fieldInfos = CreateAndWriteFieldInfos(dir, name);
+
+            FieldInfos readIn = ReadFieldInfos(dir, name);
+            Assert.IsTrue(fieldInfos.Count == readIn.Count);
+            FieldInfo info = readIn.FieldInfo("textField1");
+            Assert.IsTrue(info != null);
+            Assert.IsTrue(info.HasVectors == false);
+            Assert.IsTrue(info.OmitsNorms == false);
+
+            info = readIn.FieldInfo("textField2");
+            Assert.IsTrue(info != null);
+            Assert.IsTrue(info.OmitsNorms == false);
+
+            info = readIn.FieldInfo("textField3");
+            Assert.IsTrue(info != null);
+            Assert.IsTrue(info.HasVectors == false);
+            Assert.IsTrue(info.OmitsNorms == true);
+
+            info = readIn.FieldInfo("omitNorms");
+            Assert.IsTrue(info != null);
+            Assert.IsTrue(info.HasVectors == false);
+            Assert.IsTrue(info.OmitsNorms == true);
+
+            dir.Dispose();
+        }
+
+        [Test]
+        public virtual void TestReadOnly()
+        {
+            string name = "testFile";
+            Directory dir = NewDirectory();
+            FieldInfos fieldInfos = CreateAndWriteFieldInfos(dir, name);
+            FieldInfos readOnly = ReadFieldInfos(dir, name);
+            AssertReadOnly(readOnly, fieldInfos);
+            dir.Dispose();
+        }
+
+        private void AssertReadOnly(FieldInfos readOnly, FieldInfos modifiable)
+        {
+            Assert.AreEqual(modifiable.Count, readOnly.Count);
+            // assert we can iterate
+            foreach (FieldInfo fi in readOnly)
+            {
+                Assert.AreEqual(fi.Name, modifiable.FieldInfo(fi.Number).Name);
+            }
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestFieldsReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestFieldsReader.cs b/src/Lucene.Net.Tests/Index/TestFieldsReader.cs
new file mode 100644
index 0000000..e539561
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestFieldsReader.cs
@@ -0,0 +1,286 @@
+using System;
+using System.Collections.Generic;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+    using NUnit.Framework;
+    using System.IO;
+    using BaseDirectory = Lucene.Net.Store.BaseDirectory;
+    using BufferedIndexInput = Lucene.Net.Store.BufferedIndexInput;
+    using Directory = Lucene.Net.Store.Directory;
+    using Document = Documents.Document;
+    using DocumentStoredFieldVisitor = DocumentStoredFieldVisitor;
+    using Field = Field;
+    using IndexInput = Lucene.Net.Store.IndexInput;
+    using IndexOutput = Lucene.Net.Store.IndexOutput;
+    using IOContext = Lucene.Net.Store.IOContext;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+    using TestUtil = Lucene.Net.Util.TestUtil;
+
+    [TestFixture]
+    public class TestFieldsReader : LuceneTestCase
+    {
+        private static Directory Dir;
+        private static Document TestDoc;
+        private static FieldInfos.Builder FieldInfos = null;
+
+        /// <summary>
+        /// LUCENENET specific
+        /// Is non-static because NewIndexWriterConfig is no longer static.
+        /// </summary>
+        [OneTimeSetUp]
+        public void BeforeClass()
+        {
+            TestDoc = new Document();
+            FieldInfos = new FieldInfos.Builder();
+            DocHelper.SetupDoc(TestDoc);
+            foreach (IIndexableField field in TestDoc)
+            {
+                FieldInfos.AddOrUpdate(field.Name, field.FieldType);
+            }
+            Dir = NewDirectory();
+            IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy());
+            conf.MergePolicy.NoCFSRatio = 0.0;
+            IndexWriter writer = new IndexWriter(Dir, conf);
+            writer.AddDocument(TestDoc);
+            writer.Dispose();
+            FaultyIndexInput.DoFail = false;
+        }
+
+        [OneTimeTearDown]
+        public static void AfterClass()
+        {
+            Dir.Dispose();
+            Dir = null;
+            FieldInfos = null;
+            TestDoc = null;
+        }
+
+        [Test]
+        public virtual void Test()
+        {
+            Assert.IsTrue(Dir != null);
+            Assert.IsTrue(FieldInfos != null);
+            IndexReader reader = DirectoryReader.Open(Dir);
+            Document doc = reader.Document(0);
+            Assert.IsTrue(doc != null);
+            Assert.IsTrue(doc.GetField(DocHelper.TEXT_FIELD_1_KEY) != null);
+
+            Field field = (Field)doc.GetField(DocHelper.TEXT_FIELD_2_KEY);
+            Assert.IsTrue(field != null);
+            Assert.IsTrue(field.FieldType.StoreTermVectors);
+
+            Assert.IsFalse(field.FieldType.OmitNorms);
+            Assert.IsTrue(field.FieldType.IndexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+
+            field = (Field)doc.GetField(DocHelper.TEXT_FIELD_3_KEY);
+            Assert.IsTrue(field != null);
+            Assert.IsFalse(field.FieldType.StoreTermVectors);
+            Assert.IsTrue(field.FieldType.OmitNorms);
+            Assert.IsTrue(field.FieldType.IndexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+
+            field = (Field)doc.GetField(DocHelper.NO_TF_KEY);
+            Assert.IsTrue(field != null);
+            Assert.IsFalse(field.FieldType.StoreTermVectors);
+            Assert.IsFalse(field.FieldType.OmitNorms);
+            Assert.IsTrue(field.FieldType.IndexOptions == IndexOptions.DOCS_ONLY);
+
+            DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(DocHelper.TEXT_FIELD_3_KEY);
+            reader.Document(0, visitor);
+            IList<IIndexableField> fields = visitor.Document.Fields;
+            Assert.AreEqual(1, fields.Count);
+            Assert.AreEqual(DocHelper.TEXT_FIELD_3_KEY, fields[0].Name);
+            reader.Dispose();
+        }
+
+        public class FaultyFSDirectory : BaseDirectory
+        {
+            internal Directory FsDir;
+
+            public FaultyFSDirectory(DirectoryInfo dir)
+            {
+                FsDir = NewFSDirectory(dir);
+                m_lockFactory = FsDir.LockFactory;
+            }
+
+            public override IndexInput OpenInput(string name, IOContext context)
+            {
+                return new FaultyIndexInput(FsDir.OpenInput(name, context));
+            }
+
+            public override string[] ListAll()
+            {
+                return FsDir.ListAll();
+            }
+
+            [Obsolete("this method will be removed in 5.0")]
+            public override bool FileExists(string name)
+            {
+                return FsDir.FileExists(name);
+            }
+
+            public override void DeleteFile(string name)
+            {
+                FsDir.DeleteFile(name);
+            }
+
+            public override long FileLength(string name)
+            {
+                return FsDir.FileLength(name);
+            }
+
+            public override IndexOutput CreateOutput(string name, IOContext context)
+            {
+                return FsDir.CreateOutput(name, context);
+            }
+
+            public override void Sync(ICollection<string> names)
+            {
+                FsDir.Sync(names);
+            }
+
+            public override void Dispose()
+            {
+                FsDir.Dispose();
+            }
+        }
+
+        private class FaultyIndexInput : BufferedIndexInput
+        {
+            internal IndexInput @delegate;
+            internal static bool DoFail;
+            internal int Count;
+
+            internal FaultyIndexInput(IndexInput @delegate)
+                : base("FaultyIndexInput(" + @delegate + ")", BufferedIndexInput.BUFFER_SIZE)
+            {
+                this.@delegate = @delegate;
+            }
+
+            internal virtual void SimOutage()
+            {
+                if (DoFail && Count++ % 2 == 1)
+                {
+                    throw new IOException("Simulated network outage");
+                }
+            }
+
+            protected override void ReadInternal(byte[] b, int offset, int length)
+            {
+                SimOutage();
+                @delegate.Seek(FilePointer);
+                @delegate.ReadBytes(b, offset, length);
+            }
+
+            protected override void SeekInternal(long pos)
+            {
+            }
+
+            public override long Length
+            {
+                get { return @delegate.Length; }
+            }
+
+            public override void Dispose()
+            {
+                @delegate.Dispose();
+            }
+
+            public override object Clone()
+            {
+                FaultyIndexInput i = new FaultyIndexInput((IndexInput)@delegate.Clone());
+                // seek the clone to our current position
+                try
+                {
+                    i.Seek(FilePointer);
+                }
+#pragma warning disable 168
+                catch (IOException e)
+#pragma warning restore 168
+                {
+                    throw new Exception();
+                }
+                return i;
+            }
+        }
+
+        // LUCENE-1262
+        [Test]
+        public virtual void TestExceptions()
+        {
+            DirectoryInfo indexDir = CreateTempDir("testfieldswriterexceptions");
+
+            try
+            {
+                Directory dir = new FaultyFSDirectory(indexDir);
+                IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.CREATE);
+                IndexWriter writer = new IndexWriter(dir, iwc);
+                for (int i = 0; i < 2; i++)
+                {
+                    writer.AddDocument(TestDoc);
+                }
+                writer.ForceMerge(1);
+                writer.Dispose();
+
+                IndexReader reader = DirectoryReader.Open(dir);
+
+                FaultyIndexInput.DoFail = true;
+
+                bool exc = false;
+
+                for (int i = 0; i < 2; i++)
+                {
+                    try
+                    {
+                        reader.Document(i);
+                    }
+#pragma warning disable 168
+                    catch (IOException ioe)
+#pragma warning restore 168
+                    {
+                        // expected
+                        exc = true;
+                    }
+                    try
+                    {
+                        reader.Document(i);
+                    }
+#pragma warning disable 168
+                    catch (IOException ioe)
+#pragma warning restore 168
+                    {
+                        // expected
+                        exc = true;
+                    }
+                }
+                Assert.IsTrue(exc);
+                reader.Dispose();
+                dir.Dispose();
+            }
+            finally
+            {
+                System.IO.Directory.Delete(indexDir.FullName, true);
+            }
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestFilterAtomicReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestFilterAtomicReader.cs b/src/Lucene.Net.Tests/Index/TestFilterAtomicReader.cs
new file mode 100644
index 0000000..f22c0ee
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestFilterAtomicReader.cs
@@ -0,0 +1,224 @@
+using System;
+using System.Linq;
+using System.Reflection;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+    using NUnit.Framework;
+    using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper;
+    using IBits = Lucene.Net.Util.IBits;
+    using BytesRef = Lucene.Net.Util.BytesRef;
+    using Directory = Lucene.Net.Store.Directory;
+    using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator;
+    using Document = Documents.Document;
+    using Field = Field;
+    using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+
+    [TestFixture]
+    public class TestFilterAtomicReader : LuceneTestCase
+    {
+        private class TestReader : FilterAtomicReader
+        {
+            /// <summary>
+            /// Filter that only permits terms containing 'e'. </summary>
+            private class TestFields : FilterFields
+            {
+                internal TestFields(Fields @in)
+                    : base(@in)
+                {
+                }
+
+                public override Terms GetTerms(string field)
+                {
+                    return new TestTerms(base.GetTerms(field));
+                }
+            }
+
+            private class TestTerms : FilterTerms
+            {
+                internal TestTerms(Terms @in)
+                    : base(@in)
+                {
+                }
+
+                public override TermsEnum GetIterator(TermsEnum reuse)
+                {
+                    return new TestTermsEnum(base.GetIterator(reuse));
+                }
+            }
+
+            private class TestTermsEnum : FilterTermsEnum
+            {
+                public TestTermsEnum(TermsEnum @in)
+                    : base(@in)
+                {
+                }
+
+                /// <summary>
+                /// Scan for terms containing the letter 'e'. </summary>
+                public override BytesRef Next()
+                {
+                    BytesRef text;
+                    while ((text = m_input.Next()) != null)
+                    {
+                        if (text.Utf8ToString().IndexOf('e') != -1)
+                        {
+                            return text;
+                        }
+                    }
+                    return null;
+                }
+
+                public override DocsAndPositionsEnum DocsAndPositions(IBits liveDocs, DocsAndPositionsEnum reuse, int flags)
+                {
+                    return new TestPositions(base.DocsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum)reuse).m_input, flags));
+                }
+            }
+
+            /// <summary>
+            /// Filter that only returns odd numbered documents. </summary>
+            private class TestPositions : FilterDocsAndPositionsEnum
+            {
+                public TestPositions(DocsAndPositionsEnum input)
+                    : base(input)
+                {
+                }
+
+                /// <summary>
+                /// Scan for odd numbered documents. </summary>
+                public override int NextDoc()
+                {
+                    int doc;
+                    while ((doc = m_input.NextDoc()) != NO_MORE_DOCS)
+                    {
+                        if ((doc % 2) == 1)
+                        {
+                            return doc;
+                        }
+                    }
+                    return NO_MORE_DOCS;
+                }
+            }
+
+            public TestReader(IndexReader reader)
+                : base(SlowCompositeReaderWrapper.Wrap(reader))
+            {
+            }
+
+            public override Fields Fields
+            {
+                get { return new TestFields(base.Fields); }
+            }
+        }
+
+        /// <summary>
+        /// Tests the IndexReader.getFieldNames implementation </summary>
+        /// <exception cref="Exception"> on error </exception>
+        [Test]
+        public virtual void TestFilterIndexReader()
+        {
+            Directory directory = NewDirectory();
+
+            IndexWriter writer = new IndexWriter(directory, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+            Document d1 = new Document();
+            d1.Add(NewTextField("default", "one two", Field.Store.YES));
+            writer.AddDocument(d1);
+
+            Document d2 = new Document();
+            d2.Add(NewTextField("default", "one three", Field.Store.YES));
+            writer.AddDocument(d2);
+
+            Document d3 = new Document();
+            d3.Add(NewTextField("default", "two four", Field.Store.YES));
+            writer.AddDocument(d3);
+
+            writer.Dispose();
+
+            Directory target = NewDirectory();
+
+            // We mess with the postings so this can fail:
+            ((BaseDirectoryWrapper)target).CrossCheckTermVectorsOnClose = false;
+
+            writer = new IndexWriter(target, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+            IndexReader reader = new TestReader(DirectoryReader.Open(directory));
+            writer.AddIndexes(reader);
+            writer.Dispose();
+            reader.Dispose();
+            reader = DirectoryReader.Open(target);
+
+            TermsEnum terms = MultiFields.GetTerms(reader, "default").GetIterator(null);
+            while (terms.Next() != null)
+            {
+                Assert.IsTrue(terms.Term.Utf8ToString().IndexOf('e') != -1);
+            }
+
+            Assert.AreEqual(TermsEnum.SeekStatus.FOUND, terms.SeekCeil(new BytesRef("one")));
+
+            DocsAndPositionsEnum positions = terms.DocsAndPositions(MultiFields.GetLiveDocs(reader), null);
+            while (positions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
+            {
+                Assert.IsTrue((positions.DocID % 2) == 1);
+            }
+
+            reader.Dispose();
+            directory.Dispose();
+            target.Dispose();
+        }
+
+        private static void CheckOverrideMethods(Type clazz)
+        {
+            Type superClazz = clazz.GetTypeInfo().BaseType;
+            foreach (MethodInfo m in superClazz.GetMethods())
+            {
+                // LUCENENET specific - since we changed to using a property for Attributes rather than a method,
+                // we need to reflect that as get_Attributes here.
+                if (m.IsStatic || m.IsAbstract || m.IsFinal || /*m.Synthetic ||*/ m.Name.Equals("get_Attributes"))
+                {
+                    continue;
+                }
+                // The point of these checks is to ensure that methods that have a default
+                // impl through other methods are not overridden. this makes the number of
+                // methods to override to have a working impl minimal and prevents from some
+                // traps: for example, think about having getCoreCacheKey delegate to the
+                // filtered impl by default
+                MethodInfo subM = clazz.GetMethod(m.Name, m.GetParameters().Select(p => p.ParameterType).ToArray());
+                if (subM.DeclaringType == clazz && m.DeclaringType != typeof(object) && m.DeclaringType != subM.DeclaringType)
+                {
+                    Assert.Fail(clazz + " overrides " + m + " although it has a default impl");
+                }
+            }
+        }
+
+        [Test]
+        public virtual void TestOverrideMethods()
+        {
+            CheckOverrideMethods(typeof(FilterAtomicReader));
+            CheckOverrideMethods(typeof(FilterAtomicReader.FilterFields));
+            CheckOverrideMethods(typeof(FilterAtomicReader.FilterTerms));
+            CheckOverrideMethods(typeof(FilterAtomicReader.FilterTermsEnum));
+            CheckOverrideMethods(typeof(FilterAtomicReader.FilterDocsEnum));
+            CheckOverrideMethods(typeof(FilterAtomicReader.FilterDocsAndPositionsEnum));
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestFlex.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestFlex.cs b/src/Lucene.Net.Tests/Index/TestFlex.cs
new file mode 100644
index 0000000..01cd762
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestFlex.cs
@@ -0,0 +1,100 @@
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+    using Lucene.Net.Analysis;
+    
+
+    /*
+         * Licensed to the Apache Software Foundation (ASF) under one or more
+         * contributor license agreements.  See the NOTICE file distributed with
+         * this work for additional information regarding copyright ownership.
+         * The ASF licenses this file to You under the Apache License, Version 2.0
+         * (the "License"); you may not use this file except in compliance with
+         * the License.  You may obtain a copy of the License at
+         *
+         *     http://www.apache.org/licenses/LICENSE-2.0
+         *
+         * Unless required by applicable law or agreed to in writing, software
+         * distributed under the License is distributed on an "AS IS" BASIS,
+         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+         * See the License for the specific language governing permissions and
+         * limitations under the License.
+         */
+
+    using Lucene.Net.Store;
+    using Lucene.Net.Util;
+    using NUnit.Framework;
+    using Lucene41PostingsFormat = Lucene.Net.Codecs.Lucene41.Lucene41PostingsFormat;
+
+    [TestFixture]
+    public class TestFlex : LuceneTestCase
+    {
+        // Test non-flex API emulated on flex index
+        [Test]
+        public virtual void TestNonFlex()
+        {
+            Directory d = NewDirectory();
+
+            const int DOC_COUNT = 177;
+
+            IndexWriter w = new IndexWriter(d, (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMaxBufferedDocs(7).SetMergePolicy(NewLogMergePolicy()));
+
+            for (int iter = 0; iter < 2; iter++)
+            {
+                if (iter == 0)
+                {
+                    Documents.Document doc = new Documents.Document();
+                    doc.Add(NewTextField("field1", "this is field1", Field.Store.NO));
+                    doc.Add(NewTextField("field2", "this is field2", Field.Store.NO));
+                    doc.Add(NewTextField("field3", "aaa", Field.Store.NO));
+                    doc.Add(NewTextField("field4", "bbb", Field.Store.NO));
+                    for (int i = 0; i < DOC_COUNT; i++)
+                    {
+                        w.AddDocument(doc);
+                    }
+                }
+                else
+                {
+                    w.ForceMerge(1);
+                }
+
+                IndexReader r = w.Reader;
+
+                TermsEnum terms = MultiFields.GetTerms(r, "field3").GetIterator(null);
+                Assert.AreEqual(TermsEnum.SeekStatus.END, terms.SeekCeil(new BytesRef("abc")));
+                r.Dispose();
+            }
+
+            w.Dispose();
+            d.Dispose();
+        }
+
+        [Test]
+        public virtual void TestTermOrd()
+        {
+            Directory d = NewDirectory();
+            IndexWriter w = new IndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat())));
+            Documents.Document doc = new Documents.Document();
+            doc.Add(NewTextField("f", "a b c", Field.Store.NO));
+            w.AddDocument(doc);
+            w.ForceMerge(1);
+            DirectoryReader r = w.Reader;
+            TermsEnum terms = GetOnlySegmentReader(r).Fields.GetTerms("f").GetIterator(null);
+            Assert.IsTrue(terms.Next() != null);
+            try
+            {
+                Assert.AreEqual(0, terms.Ord);
+            }
+#pragma warning disable 168
+            catch (System.NotSupportedException uoe)
+#pragma warning restore 168
+            {
+                // ok -- codec is not required to support this op
+            }
+            r.Dispose();
+            w.Dispose();
+            d.Dispose();
+        }
+    }
+}
\ No newline at end of file


Mime
View raw message