lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dougs...@apache.org
Subject svn commit: r798995 [8/35] - in /incubator/lucene.net/trunk/C#/src: Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Analysis/Standard/ Lucene.Net/Document/ Lucene.Net/Index/ Lucene.Net/QueryParser/ Lucene.Net/Search/ Lucene.Net/Search/Function/ Lucene.Net/...
Date Wed, 29 Jul 2009 18:04:24 GMT
Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/FreqProxTermsWriter.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriter.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriter.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,423 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System.Collections.Generic;
+
+using IndexInput = Lucene.Net.Store.IndexInput;
+using IndexOutput = Lucene.Net.Store.IndexOutput;
+using UnicodeUtil = Lucene.Net.Util.UnicodeUtil;
+
+namespace Lucene.Net.Index
+{
+    internal sealed class FreqProxTermsWriter : TermsHashConsumer
+    {
+
+        internal override TermsHashConsumerPerThread addThread(TermsHashPerThread perThread)
+        {
+            return new FreqProxTermsWriterPerThread(perThread);
+        }
+
+        internal override void createPostings(RawPostingList[] postings, int start, int count)
+        {
+            int end = start + count;
+            for (int i = start; i < end; i++)
+                postings[i] = new PostingList();
+        }
+
+        private static int compareText(char[] text1, int pos1, char[] text2, int pos2)
+        {
+            while (true)
+            {
+                char c1 = text1[pos1++];
+                char c2 = text2[pos2++];
+                if (c1 != c2)
+                {
+                    if (0xffff == c2)
+                        return 1;
+                    else if (0xffff == c1)
+                        return -1;
+                    else
+                        return c1 - c2;
+                }
+                else if (0xffff == c1)
+                    return 0;
+            }
+        }
+
+        internal override void closeDocStore(DocumentsWriter.FlushState state) { }
+        internal override void Abort() { }
+
+
+        // TODO: would be nice to factor out morme of this, eg the
+        // FreqProxFieldMergeState, and code to visit all Fields
+        // under the same FieldInfo together, up into TermsHash*.
+        // Other writers would presumably share alot of this...
+
+        internal override void flush(IDictionary<object, object> threadsAndFields, DocumentsWriter.FlushState state)
+        {
+
+            // Gather all FieldData's that have postings, across all
+            // ThreadStates
+            List<object> allFields = new List<object>();
+
+            IEnumerator<KeyValuePair<object, object>> it = threadsAndFields.GetEnumerator();
+            while (it.MoveNext())
+            {
+
+                KeyValuePair<object, object> entry = (KeyValuePair<object, object>)it.Current;
+
+                ICollection<object> fields = (ICollection<object>)entry.Value;
+
+                IEnumerator<object> fieldsIt = fields.GetEnumerator();
+
+                while (fieldsIt.MoveNext())
+                {
+                    FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField)fieldsIt.Current;
+                    if (perField.termsHashPerField.numPostings > 0)
+                        allFields.Add(perField);
+                }
+            }
+
+            // Sort by field name
+            allFields.Sort();
+            int numAllFields = allFields.Count;
+
+            TermInfosWriter termsOut = new TermInfosWriter(state.directory,
+                                                                 state.segmentName,
+                                                                 fieldInfos,
+                                                                 state.docWriter.writer.GetTermIndexInterval());
+
+            IndexOutput freqOut = state.directory.CreateOutput(state.SegmentFileName(IndexFileNames.FREQ_EXTENSION));
+            IndexOutput proxOut;
+
+            if (fieldInfos.HasProx())
+                proxOut = state.directory.CreateOutput(state.SegmentFileName(IndexFileNames.PROX_EXTENSION));
+            else
+                proxOut = null;
+
+            DefaultSkipListWriter skipListWriter = new DefaultSkipListWriter(termsOut.skipInterval,
+                                                                                   termsOut.maxSkipLevels,
+                                                                                   state.numDocsInRAM, freqOut, proxOut);
+
+            int start = 0;
+            while (start < numAllFields)
+            {
+                FieldInfo fieldInfo = ((FreqProxTermsWriterPerField)allFields[start]).fieldInfo;
+                string fieldName = fieldInfo.name;
+
+                int end = start + 1;
+                while (end < numAllFields && ((FreqProxTermsWriterPerField)allFields[end]).fieldInfo.name.Equals(fieldName))
+                    end++;
+
+                FreqProxTermsWriterPerField[] fields = new FreqProxTermsWriterPerField[end - start];
+                for (int i = start; i < end; i++)
+                {
+                    fields[i - start] = (FreqProxTermsWriterPerField)allFields[i];
+
+                    // Aggregate the storePayload as seen by the same
+                    // field across multiple threads
+                    fieldInfo.storePayloads |= fields[i - start].hasPayloads;
+                }
+
+                // If this field has postings then add them to the
+                // segment
+                AppendPostings(state, fields, termsOut, freqOut, proxOut, skipListWriter);
+
+                for (int i = 0; i < fields.Length; i++)
+                {
+                    TermsHashPerField perField = fields[i].termsHashPerField;
+                    int numPostings = perField.numPostings;
+                    perField.reset();
+                    perField.shrinkHash(numPostings);
+                    fields[i].reset();
+                }
+
+                start = end;
+            }
+
+            it = threadsAndFields.GetEnumerator();
+            while (it.MoveNext())
+            {
+                KeyValuePair<object, object> entry = (KeyValuePair<object, object>)it.Current;
+                FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread)entry.Key;
+                perThread.termsHashPerThread.reset(true);
+            }
+
+            freqOut.Close();
+            if (proxOut != null)
+            {
+                state.flushedFiles[state.SegmentFileName(IndexFileNames.PROX_EXTENSION)] = state.SegmentFileName(IndexFileNames.PROX_EXTENSION);
+                proxOut.Close();
+            }
+            termsOut.Close();
+
+            // Record all files we have flushed
+            state.flushedFiles[state.SegmentFileName(IndexFileNames.FIELD_INFOS_EXTENSION)] = state.SegmentFileName(IndexFileNames.FIELD_INFOS_EXTENSION);
+            state.flushedFiles[state.SegmentFileName(IndexFileNames.FREQ_EXTENSION)] = state.SegmentFileName(IndexFileNames.FREQ_EXTENSION);
+            state.flushedFiles[state.SegmentFileName(IndexFileNames.TERMS_EXTENSION)] = state.SegmentFileName(IndexFileNames.TERMS_EXTENSION);
+            state.flushedFiles[state.SegmentFileName(IndexFileNames.TERMS_INDEX_EXTENSION)] = state.SegmentFileName(IndexFileNames.TERMS_INDEX_EXTENSION);
+        }
+
+        byte[] copyByteBuffer = new byte[4096];
+
+        /** Copy numBytes from srcIn to destIn */
+        void copyBytes(IndexInput srcIn, IndexOutput destIn, long numBytes)
+        {
+            // TODO: we could do this more efficiently (save a copy)
+            // because it's always from a ByteSliceReader ->
+            // IndexOutput
+            while (numBytes > 0)
+            {
+                int chunk;
+                if (numBytes > 4096)
+                    chunk = 4096;
+                else
+                    chunk = (int)numBytes;
+                srcIn.ReadBytes(copyByteBuffer, 0, chunk);
+                destIn.WriteBytes(copyByteBuffer, 0, chunk);
+                numBytes -= chunk;
+            }
+        }
+
+        /* Walk through all unique text tokens (Posting
+         * instances) found in this field and serialize them
+         * into a single RAM segment. */
+        void AppendPostings(DocumentsWriter.FlushState flushState,
+                            FreqProxTermsWriterPerField[] fields,
+                            TermInfosWriter termsOut,
+                            IndexOutput freqOut,
+                            IndexOutput proxOut,
+                            DefaultSkipListWriter skipListWriter)
+        {
+
+            int fieldNumber = fields[0].fieldInfo.number;
+            int numFields = fields.Length;
+
+            FreqProxFieldMergeState[] mergeStates = new FreqProxFieldMergeState[numFields];
+
+            for (int i = 0; i < numFields; i++)
+            {
+                FreqProxFieldMergeState fms = mergeStates[i] = new FreqProxFieldMergeState(fields[i]);
+
+                System.Diagnostics.Debug.Assert(fms.field.fieldInfo == fields[0].fieldInfo);
+
+                // Should always be true
+                bool result = fms.nextTerm();
+                System.Diagnostics.Debug.Assert(result);
+            }
+
+            int skipInterval = termsOut.skipInterval;
+            bool currentFieldOmitTf = fields[0].fieldInfo.omitTf;
+
+            // If current field omits tf then it cannot store
+            // payloads.  We silently drop the payloads in this case:
+            bool currentFieldStorePayloads = currentFieldOmitTf ? false : fields[0].fieldInfo.storePayloads;
+
+            FreqProxFieldMergeState[] termStates = new FreqProxFieldMergeState[numFields];
+
+            while (numFields > 0)
+            {
+
+                // Get the next term to merge
+                termStates[0] = mergeStates[0];
+                int numToMerge = 1;
+
+                for (int i = 1; i < numFields; i++)
+                {
+                    char[] text = mergeStates[i].text;
+                    int textOffset = mergeStates[i].textOffset;
+                    int cmp = compareText(text, textOffset, termStates[0].text, termStates[0].textOffset);
+
+                    if (cmp < 0)
+                    {
+                        termStates[0] = mergeStates[i];
+                        numToMerge = 1;
+                    }
+                    else if (cmp == 0)
+                        termStates[numToMerge++] = mergeStates[i];
+                }
+
+                int df = 0;
+                int lastPayloadLength = -1;
+
+                int lastDoc = 0;
+
+                char[] text_Renamed = termStates[0].text;
+                int start = termStates[0].textOffset;
+
+                long freqPointer = freqOut.GetFilePointer();
+                long proxPointer;
+                if (proxOut != null)
+                    proxPointer = proxOut.GetFilePointer();
+                else
+                    proxPointer = 0;
+
+                skipListWriter.ResetSkip();
+
+                // Now termStates has numToMerge FieldMergeStates
+                // which all share the same term.  Now we must
+                // interleave the docID streams.
+                while (numToMerge > 0)
+                {
+
+                    if ((++df % skipInterval) == 0)
+                    {
+                        skipListWriter.SetSkipData(lastDoc, currentFieldStorePayloads, lastPayloadLength);
+                        skipListWriter.BufferSkip(df);
+                    }
+
+                    FreqProxFieldMergeState minState = termStates[0];
+                    for (int i = 1; i < numToMerge; i++)
+                        if (termStates[i].docID < minState.docID)
+                            minState = termStates[i];
+
+                    int doc = minState.docID;
+                    int termDocFreq = minState.termFreq;
+
+                    System.Diagnostics.Debug.Assert(doc < flushState.numDocsInRAM);
+                    System.Diagnostics.Debug.Assert(doc > lastDoc || df == 1);
+
+                    ByteSliceReader prox = minState.prox;
+
+                    // Carefully copy over the prox + payload info,
+                    // changing the format to match Lucene's segment
+                    // format.
+                    if (!currentFieldOmitTf)
+                    {
+                        // omitTf == false so we do write positions & payload          
+                        System.Diagnostics.Debug.Assert(proxOut != null);
+                        for (int j = 0; j < termDocFreq; j++)
+                        {
+                            int code = prox.ReadVInt();
+                            if (currentFieldStorePayloads)
+                            {
+                                int payloadLength;
+                                if ((code & 1) != 0)
+                                {
+                                    // This position has a payload
+                                    payloadLength = prox.ReadVInt();
+                                }
+                                else
+                                    payloadLength = 0;
+                                if (payloadLength != lastPayloadLength)
+                                {
+                                    proxOut.WriteVInt(code | 1);
+                                    proxOut.WriteVInt(payloadLength);
+                                    lastPayloadLength = payloadLength;
+                                }
+                                else
+                                    proxOut.WriteVInt(code & (~1));
+                                if (payloadLength > 0)
+                                    copyBytes(prox, proxOut, payloadLength);
+                            }
+                            else
+                            {
+                                System.Diagnostics.Debug.Assert(0 == (code & 1));
+                                proxOut.WriteVInt(code >> 1);
+                            }
+                        } //End for
+
+                        int newDocCode = (doc - lastDoc) << 1;
+
+                        if (1 == termDocFreq)
+                        {
+                            freqOut.WriteVInt(newDocCode | 1);
+                        }
+                        else
+                        {
+                            freqOut.WriteVInt(newDocCode);
+                            freqOut.WriteVInt(termDocFreq);
+                        }
+                    }
+                    else
+                    {
+                        // omitTf==true: we store only the docs, without
+                        // term freq, positions, payloads
+                        freqOut.WriteVInt(doc - lastDoc);
+                    }
+
+                    lastDoc = doc;
+
+                    if (!minState.nextDoc())
+                    {
+
+                        // Remove from termStates
+                        int upto = 0;
+                        for (int i = 0; i < numToMerge; i++)
+                            if (termStates[i] != minState)
+                                termStates[upto++] = termStates[i];
+                        numToMerge--;
+                        System.Diagnostics.Debug.Assert(upto == numToMerge);
+
+                        // Advance this state to the next term
+
+                        if (!minState.nextTerm())
+                        {
+                            // OK, no more terms, so remove from mergeStates
+                            // as well
+                            upto = 0;
+                            for (int i = 0; i < numFields; i++)
+                                if (mergeStates[i] != minState)
+                                    mergeStates[upto++] = mergeStates[i];
+                            numFields--;
+                            System.Diagnostics.Debug.Assert(upto == numFields);
+                        }
+                    }
+                }
+
+                System.Diagnostics.Debug.Assert(df > 0);
+
+                // Done merging this term
+
+                long skipPointer = skipListWriter.WriteSkip(freqOut);
+
+                // Write term
+                termInfo.Set(df, freqPointer, proxPointer, (int)(skipPointer - freqPointer));
+
+                // TODO: we could do this incrementally
+                UnicodeUtil.UTF16toUTF8(text_Renamed, start, termsUTF8);
+
+                // TODO: we could save O(n) re-scan of the term by
+                // computing the shared prefix with the last term
+                // while during the UTF8 encoding
+                termsOut.Add(fieldNumber,
+                             termsUTF8.result,
+                             termsUTF8.length,
+                             termInfo);
+            }
+        }
+
+        private readonly TermInfo termInfo = new TermInfo(); // minimize consing
+
+        internal readonly UnicodeUtil.UTF8Result termsUTF8 = new UnicodeUtil.UTF8Result();
+
+        void files(ICollection<object> files) { }
+
+        internal sealed class PostingList : RawPostingList
+        {
+            internal int docFreq;                                    // # times this term occurs in the current doc
+            internal int lastDocID;                                  // Last docID where this term occurred
+            internal int lastDocCode;                                // Code for prior doc
+            internal int lastPosition;                               // Last position where this term occurred
+        }
+
+        internal override int bytesPerPosting()
+        {
+            return RawPostingList.BYTES_SIZE + 4 * DocumentsWriter.INT_NUM_BYTE;
+        }
+    }
+}

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerField.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/FreqProxTermsWriterPerField.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerField.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerField.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Fieldable = Lucene.Net.Documents.Fieldable;
+using Token = Lucene.Net.Analysis.Token;
+
+namespace Lucene.Net.Index
+{
+    // TODO: break into separate freq and prox writers as
+    // codecs; make separate container (tii/tis/skip/*) that can
+    // be configured as any number of files 1..N
+    internal sealed class FreqProxTermsWriterPerField : TermsHashConsumerPerField, System.IComparable
+    {
+        internal readonly FreqProxTermsWriterPerThread perThread;
+        internal readonly TermsHashPerField termsHashPerField;
+        internal readonly FieldInfo fieldInfo;
+        internal readonly DocumentsWriter.DocState docState;
+        internal readonly DocInverter.FieldInvertState fieldState;
+        internal bool omitTf;
+
+        public FreqProxTermsWriterPerField(TermsHashPerField termsHashPerField, FreqProxTermsWriterPerThread perThread, FieldInfo fieldInfo)
+        {
+            this.termsHashPerField = termsHashPerField;
+            this.perThread = perThread;
+            this.fieldInfo = fieldInfo;
+            docState = termsHashPerField.docState;
+            fieldState = termsHashPerField.fieldState;
+            omitTf = fieldInfo.omitTf;
+        }
+
+        internal override int getStreamCount()
+        {
+            if (fieldInfo.omitTf)
+                return 1;
+            else
+                return 2;
+        }
+
+        internal override void finish() { }
+
+        internal bool hasPayloads;
+
+        internal override void skippingLongTerm(Token t) { }
+
+        public int CompareTo(object other0)
+        {
+            FreqProxTermsWriterPerField other = (FreqProxTermsWriterPerField)other0;
+            return string.CompareOrdinal(fieldInfo.name, other.fieldInfo.name);
+        }
+
+        internal void reset()
+        {
+            // Record, up front, whether our in-RAM format will be
+            // with or without term freqs:
+            omitTf = fieldInfo.omitTf;
+        }
+
+        internal override bool start(Fieldable[] fields, int count)
+        {
+            for (int i = 0; i < count; i++)
+                if (fields[i].IsIndexed())
+                    return true;
+            return false;
+        }
+
+        internal void writeProx(Token t, FreqProxTermsWriter.PostingList p, int proxCode)
+        {
+            Payload payload = t.GetPayload();
+            if (payload != null && payload.length > 0)
+            {
+                termsHashPerField.writeVInt(1, (proxCode << 1) | 1);
+                termsHashPerField.writeVInt(1, payload.length);
+                termsHashPerField.writeBytes(1, payload.data, payload.offset, payload.length);
+                hasPayloads = true;
+            }
+            else
+                termsHashPerField.writeVInt(1, proxCode << 1);
+            p.lastPosition = fieldState.position;
+        }
+
+        internal override void newTerm(Token t, RawPostingList p0)
+        {
+            // First time we're seeing this term since the last
+            // flush
+            System.Diagnostics.Debug.Assert(docState.TestPoint("FreqProxTermsWriterPerField.newTerm start"));
+            FreqProxTermsWriter.PostingList p = (FreqProxTermsWriter.PostingList)p0;
+            p.lastDocID = docState.docID;
+            if (omitTf)
+            {
+                p.lastDocCode = docState.docID;
+            }
+            else
+            {
+                p.lastDocCode = docState.docID << 1;
+                p.docFreq = 1;
+                writeProx(t, p, fieldState.position);
+            }
+        }
+
+        internal override void addTerm(Token t, RawPostingList p0)
+        {
+
+            System.Diagnostics.Debug.Assert(docState.TestPoint("FreqProxTermsWriterPerField.addTerm start"));
+
+            FreqProxTermsWriter.PostingList p = (FreqProxTermsWriter.PostingList)p0;
+
+            System.Diagnostics.Debug.Assert(omitTf || p.docFreq > 0);
+
+            if (omitTf)
+            {
+                if (docState.docID != p.lastDocID)
+                {
+                    System.Diagnostics.Debug.Assert(docState.docID > p.lastDocID);
+                    termsHashPerField.writeVInt(0, p.lastDocCode);
+                    p.lastDocCode = docState.docID - p.lastDocID;
+                    p.lastDocID = docState.docID;
+                }
+            }
+            else
+            {
+                if (docState.docID != p.lastDocID)
+                {
+                    System.Diagnostics.Debug.Assert(docState.docID > p.lastDocID);
+                    // Term not yet seen in the current doc but previously
+                    // seen in other doc(s) since the last flush
+
+                    // Now that we know doc freq for previous doc,
+                    // write it & lastDocCode
+                    if (1 == p.docFreq)
+                        termsHashPerField.writeVInt(0, p.lastDocCode | 1);
+                    else
+                    {
+                        termsHashPerField.writeVInt(0, p.lastDocCode);
+                        termsHashPerField.writeVInt(0, p.docFreq);
+                    }
+                    p.docFreq = 1;
+                    p.lastDocCode = (docState.docID - p.lastDocID) << 1;
+                    p.lastDocID = docState.docID;
+                    writeProx(t, p, fieldState.position);
+                }
+                else
+                {
+                    p.docFreq++;
+                    writeProx(t, p, fieldState.position - p.lastPosition);
+                }
+            }
+        }
+
+        public void abort() { }
+    }
+}
\ No newline at end of file

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerThread.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/FreqProxTermsWriterPerThread.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerThread.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerThread.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Lucene.Net.Index
+{
+    internal sealed class FreqProxTermsWriterPerThread : TermsHashConsumerPerThread
+    {
+        internal readonly TermsHashPerThread termsHashPerThread;
+        internal readonly DocumentsWriter.DocState docState;
+
+        public FreqProxTermsWriterPerThread(TermsHashPerThread perThread)
+        {
+            docState = perThread.docState;
+            termsHashPerThread = perThread;
+        }
+
+        public override TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo)
+        {
+            return new FreqProxTermsWriterPerField(termsHashPerField, this, fieldInfo);
+        }
+
+        internal override void startDocument()
+        {
+        }
+
+        internal override DocumentsWriter.DocWriter finishDocument()
+        {
+            return null;
+        }
+
+        public override void abort() { }
+    }
+}

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommit.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/IndexCommit.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommit.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommit.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using Directory = Lucene.Net.Store.Directory;
+
+namespace Lucene.Net.Index
+{
+    /// <summary>
+    /// Deprecated.  Please subclass Indexcommit class instead.
+    /// </summary>
+    public abstract class IndexCommit : IndexCommitPoint
+    {
+
+        /**
+         * Get the segments file (<code>segments_N</code>) associated 
+         * with this commit point.
+         */
+        public abstract string GetSegmentsFileName();
+
+        /**
+         * Returns all index files referenced by this commit point.
+         */
+        public abstract System.Collections.Generic.ICollection<string> GetFileNames();
+
+        /**
+         * Returns the {@link Directory} for the index.
+         */
+        public abstract Directory GetDirectory();
+
+        /**
+         * Delete this commit point.  This only applies when using
+         * the commit point in the context of IndexWriter's
+         * IndexDeletionPolicy.
+         * <p>
+         * Upon calling this, the writer is notified that this commit 
+         * point should be deleted. 
+         * <p>
+         * Decision that a commit-point should be deleted is taken by the {@link IndexDeletionPolicy} in effect
+         * and therefore this should only be called by its {@link IndexDeletionPolicy#onInit onInit()} or 
+         * {@link IndexDeletionPolicy#onCommit onCommit()} methods.
+        */
+        public virtual void Delete()
+        {
+            throw new System.Exception("This IndexCommit does not support this method.");
+        }
+
+        public virtual bool IsDeleted()
+        {
+            throw new System.Exception("This IndexCommit does not support this method.");
+        }
+
+        /**
+         * Returns true if this commit is an optimized index.
+         */
+        public virtual bool IsOptimized()
+        {
+            throw new System.Exception("This IndexCommit does not support this method.");
+        }
+
+        /**
+         * Two IndexCommits are equal if both their Directory and versions are equal.
+         */
+        public override bool Equals(object other)
+        {
+            if (other is IndexCommit)
+            {
+                IndexCommit otherCommit = (IndexCommit)other;
+                return otherCommit.GetDirectory().Equals(GetDirectory()) && otherCommit.GetVersion() == GetVersion();
+            }
+            else
+                return false;
+        }
+
+        public override int GetHashCode()
+        {
+            return GetDirectory().GetHashCode() + GetSegmentsFileName().GetHashCode();
+        }
+
+        /** Returns the version for this IndexCommit.  This is the
+            same value that {@link IndexReader#getVersion} would
+            return if it were opened on this commit. */
+        public virtual long GetVersion()
+        {
+            throw new System.Exception("This IndexCommit does not support this method.");
+        }
+
+        /** Returns the generation (the _N in segments_N) for this
+            IndexCommit */
+        public virtual long GetGeneration()
+        {
+            throw new System.Exception("This IndexCommit does not support this method.");
+        }
+
+        /** Convenience method that returns the last modified time
+         *  of the segments_N file corresponding to this index
+         *  commit, equivalent to
+         *  getDirectory().fileModified(getSegmentsFileName()). */
+        public long GetTimestamp()
+        {
+            return GetDirectory().FileModified(GetSegmentsFileName());
+        }
+    }
+}

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommitPoint.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/IndexCommitPoint.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommitPoint.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommitPoint.cs Wed Jul 29 18:04:12 2009
@@ -29,7 +29,6 @@
 /// index commit point would have a larger N.
 /// </summary>
 
-using System;
 
 namespace Lucene.Net.Index
 {
@@ -40,10 +39,10 @@
 		/// <summary> Get the segments file (<code>segments_N</code>) associated 
 		/// with this commit point.
 		/// </summary>
-		System.String GetSegmentsFileName();
+		string GetSegmentsFileName();
 		
 		/// <summary> Returns all index files referenced by this commit point.</summary>
-		System.Collections.ICollection GetFileNames();
+		System.Collections.Generic.ICollection<string> GetFileNames();
 		
 		/// <summary> Delete this commit point.
 		/// <p>
@@ -54,6 +53,6 @@
 		/// and therefore this should only be called by its {@link IndexDeletionPolicy#onInit onInit()} or 
 		/// {@link IndexDeletionPolicy#onCommit onCommit()} methods.
 		/// </summary>
-		void  Delete();
+		void Delete();
 	}
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexDeletionPolicy.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/IndexDeletionPolicy.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexDeletionPolicy.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexDeletionPolicy.cs Wed Jul 29 18:04:12 2009
@@ -15,17 +15,17 @@
  * limitations under the License.
  */
 
-using System;
+using System.Collections.Generic;
 
 namespace Lucene.Net.Index
 {
 	
-	/// <summary> <p>Expert: policy for deletion of stale {@link IndexCommitPoint index commits}. 
+	/// <summary> <p>Expert: policy for deletion of stale {@link IndexCommit index commits}. 
 	/// 
 	/// <p>Implement this interface, and pass it to one
 	/// of the {@link IndexWriter} or {@link IndexReader}
 	/// constructors, to customize when older
-	/// {@link IndexCommitPoint point-in-time commits}
+	/// {@link IndexCommit point-in-time commits}
 	/// are deleted from the index directory.  The default deletion policy
 	/// is {@link KeepOnlyLastCommitDeletionPolicy}, which always
 	/// removes old commits as soon as a new commit is done (this
@@ -57,8 +57,8 @@
 		/// <p>The writer locates all index commits present in the 
 		/// index directory and calls this method.  The policy may 
 		/// choose to delete some of the commit points, doing so by
-		/// calling method {@link IndexCommitPoint#delete delete()} 
-		/// of {@link IndexCommitPoint}.</p>
+		/// calling method {@link IndexCommit#delete delete()} 
+		/// of {@link IndexCommit}.</p>
 		/// 
 		/// <p><u>Note:</u> the last CommitPoint is the most recent one,
 		/// i.e. the "front index state". Be careful not to delete it,
@@ -67,18 +67,18 @@
 		/// 
 		/// </summary>
 		/// <param name="commits">List of current 
-		/// {@link IndexCommitPoint point-in-time commits},
+		/// {@link IndexCommit point-in-time commits},
 		/// sorted by age (the 0th one is the oldest commit).
 		/// </param>
-		void  OnInit(System.Collections.IList commits);
+		void  OnInit(List<IndexCommitPoint> commits);
 		
 		/// <summary> <p>This is called each time the writer completed a commit.
 		/// This gives the policy a chance to remove old commit points
 		/// with each commit.</p>
 		/// 
 		/// <p>The policy may now choose to delete old commit points 
-		/// by calling method {@link IndexCommitPoint#delete delete()} 
-		/// of {@link IndexCommitPoint}.</p>
+		/// by calling method {@link IndexCommit#delete delete()} 
+		/// of {@link IndexCommit}.</p>
 		/// 
 		/// <p>If writer has <code>autoCommit = true</code> then
 		/// this method will in general be called many times during
@@ -94,9 +94,9 @@
 		/// you can afford to lose the index content while doing that.
 		/// 
 		/// </summary>
-		/// <param name="commits">List of {@link IndexCommitPoint},
+		/// <param name="commits">List of {@link IndexCommit},
 		/// sorted by age (the 0th one is the oldest commit).
 		/// </param>
-		void  OnCommit(System.Collections.IList commits);
+		void  OnCommit(List<IndexCommitPoint> commits);
 	}
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/IndexFileDeleter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs Wed Jul 29 18:04:12 2009
@@ -15,710 +15,753 @@
  * limitations under the License.
  */
 
-using System;
+using System.Collections.Generic;
 
 using Directory = Lucene.Net.Store.Directory;
 
 namespace Lucene.Net.Index
 {
-	
-	/*
-	* This class keeps track of each SegmentInfos instance that
-	* is still "live", either because it corresponds to a 
-	* segments_N file in the Directory (a "commit", i.e. a 
-	* committed SegmentInfos) or because it's the in-memory SegmentInfos 
-	* that a writer is actively updating but has not yet committed 
-	* (currently this only applies when autoCommit=false in IndexWriter).
-	* This class uses simple reference counting to map the live
-	* SegmentInfos instances to individual files in the Directory. 
-	* 
-	* The same directory file may be referenced by more than
-	* one IndexCommitPoints, i.e. more than one SegmentInfos.
-	* Therefore we count how many commits reference each file.
-	* When all the commits referencing a certain file have been
-	* deleted, the refcount for that file becomes zero, and the
-	* file is deleted.
-	*
-	* A separate deletion policy interface
-	* (IndexDeletionPolicy) is consulted on creation (onInit)
-	* and once per commit (onCommit), to decide when a commit
-	* should be removed.
-	* 
-	* It is the business of the IndexDeletionPolicy to choose
-	* when to delete commit points.  The actual mechanics of
-	* file deletion, retrying, etc, derived from the deletion
-	* of commit points is the business of the IndexFileDeleter.
-	* 
-	* The current default deletion policy is {@link
-	* KeepOnlyLastCommitDeletionPolicy}, which removes all
-	* prior commits when a new commit has completed.  This
-	* matches the behavior before 2.2.
-	*
-	* Note that you must hold the write.lock before
-	* instantiating this class.  It opens segments_N file(s)
-	* directly with no retry logic.
-	*/
-	
-	sealed public class IndexFileDeleter
-	{
-		
-		/* Files that we tried to delete but failed (likely
-		* because they are open and we are running on Windows),
-		* so we will retry them again later: */
-		private System.Collections.IList deletable;
-		
-		/* Reference count for all files in the index.  
-		* Counts how many existing commits reference a file.
-		* Maps String to RefCount (class below) instances: */
-		private System.Collections.IDictionary refCounts = new System.Collections.Hashtable();
-		
-		/* Holds all commits (segments_N) currently in the index.
-		* This will have just 1 commit if you are using the
-		* default delete policy (KeepOnlyLastCommitDeletionPolicy).
-		* Other policies may leave commit points live for longer
-		* in which case this list would be longer than 1: */
-		private System.Collections.ArrayList commits = new System.Collections.ArrayList();
-		
-		/* Holds files we had incref'd from the previous
-		* non-commit checkpoint: */
-		private System.Collections.IList lastFiles = new System.Collections.ArrayList();
-		
-		/* Commits that the IndexDeletionPolicy have decided to delete: */
-		private System.Collections.IList commitsToDelete = new System.Collections.ArrayList();
+
+    /*
+     * This class keeps track of each SegmentInfos instance that
+     * is still "live", either because it corresponds to a
+     * segments_N file in the Directory (a "commit", i.e. a
+     * committed SegmentInfos) or because it's an in-memory
+     * SegmentInfos that a writer is actively updating but has
+     * not yet committed.  This class uses simple reference
+     * counting to map the live SegmentInfos instances to
+     * individual files in the Directory.
+     *
+     * When autoCommit=true, IndexWriter currently commits only
+     * on completion of a merge (though this may change with
+     * time: it is not a guarantee).  When autoCommit=false,
+     * IndexWriter only commits when it is closed.  Regardless
+     * of autoCommit, the user may call IndexWriter.commit() to
+     * force a blocking commit.
+     * 
+     * The same directory file may be referenced by more than
+     * one IndexCommit, i.e. more than one SegmentInfos.
+     * Therefore we count how many commits reference each file.
+     * When all the commits referencing a certain file have been
+     * deleted, the refcount for that file becomes zero, and the
+     * file is deleted.
+     *
+     * A separate deletion policy interface
+     * (IndexDeletionPolicy) is consulted on creation (onInit)
+     * and once per commit (onCommit), to decide when a commit
+     * should be removed.
+     * 
+     * It is the business of the IndexDeletionPolicy to choose
+     * when to delete commit points.  The actual mechanics of
+     * file deletion, retrying, etc, derived from the deletion
+     * of commit points is the business of the IndexFileDeleter.
+     * 
+     * The current default deletion policy is {@link
+     * KeepOnlyLastCommitDeletionPolicy}, which removes all
+     * prior commits when a new commit has completed.  This
+     * matches the behavior before 2.2.
+     *
+     * Note that you must hold the write.lock before
+     * instantiating this class.  It opens segments_N file(s)
+     * directly with no retry logic.
+     */
+    sealed public class IndexFileDeleter
+    {
+        /* Files that we tried to delete but failed (likely
+        * because they are open and we are running on Windows),
+        * so we will retry them again later: */
+        private List<string> deletable;
+
+        /* Reference count for all files in the index.  
+        * Counts how many existing commits reference a file.
+        * Maps string to RefCount (class below) instances: */
+        private Dictionary<string,RefCount> refCounts = new Dictionary<string,RefCount>();
+
+        /* Holds all commits (segments_N) currently in the index.
+        * This will have just 1 commit if you are using the
+        * default delete policy (KeepOnlyLastCommitDeletionPolicy).
+        * Other policies may leave commit points live for longer
+        * in which case this list would be longer than 1: */
+        private List<IndexCommitPoint> commits = new List<IndexCommitPoint>();
+
+        /* Holds files we had incref'd from the previous
+        * non-commit checkpoint: */
+        private List<string> lastFiles = new List<string>();
+
+        /* Commits that the IndexDeletionPolicy have decided to delete: */
+        private List<IndexCommitPoint> commitsToDelete = new List<IndexCommitPoint>();
 
         private System.IO.TextWriter infoStream;
-		private Directory directory;
-		private IndexDeletionPolicy policy;
-		private DocumentsWriter docWriter;
-		
-		/// <summary>Change to true to see details of reference counts when
-		/// infoStream != null 
-		/// </summary>
-		public static bool VERBOSE_REF_COUNTS = false;
-		
-		internal void  SetInfoStream(System.IO.TextWriter infoStream)
-		{
-			this.infoStream = infoStream;
-			if (infoStream != null)
-			{
-				Message("setInfoStream deletionPolicy=" + policy);
-			}
-		}
-		
-		private void  Message(System.String message)
-		{
-			infoStream.WriteLine("IFD [" + SupportClass.ThreadClass.Current().Name + "]: " + message);
-		}
-		
-		/// <summary> Initialize the deleter: find all previous commits in
-		/// the Directory, incref the files they reference, call
-		/// the policy to let it delete commits.  The incoming
-		/// segmentInfos must have been loaded from a commit point
-		/// and not yet modified.  This will remove any files not
-		/// referenced by any of the commits.
-		/// </summary>
-		/// <throws>  CorruptIndexException if the index is corrupt </throws>
-		/// <throws>  IOException if there is a low-level IO error </throws>
-		public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.TextWriter infoStream, DocumentsWriter docWriter)
-		{
-			
-			this.docWriter = docWriter;
-			this.infoStream = infoStream;
-			
-			if (infoStream != null)
-			{
-				Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
-			}
-			
-			this.policy = policy;
-			this.directory = directory;
-			
-			// First pass: walk the files and initialize our ref
-			// counts:
-			long currentGen = segmentInfos.GetGeneration();
-			IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
-			
-			System.String[] files = directory.List();
-			if (files == null)
-			{
-				throw new System.IO.IOException("cannot read directory " + directory + ": list() returned null");
-			}
-			
-			CommitPoint currentCommitPoint = null;
-			
-			for (int i = 0; i < files.Length; i++)
-			{
-				
-				System.String fileName = files[i];
-				
-				if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
-				{
-					
-					// Add this file to refCounts with initial count 0:
-					GetRefCount(fileName);
-					
-					if (fileName.StartsWith(IndexFileNames.SEGMENTS))
-					{
-						
-						// This is a commit (segments or segments_N), and
-						// it's valid (<= the max gen).  Load it, then
-						// incref all files it refers to:
-						if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
-						{
-							if (infoStream != null)
-							{
-								Message("init: load commit \"" + fileName + "\"");
-							}
-							SegmentInfos sis = new SegmentInfos();
-							try
-							{
-								sis.Read(directory, fileName);
-							}
-							catch (System.IO.FileNotFoundException e)
-							{
-								// LUCENE-948: on NFS (and maybe others), if
-								// you have writers switching back and forth
-								// between machines, it's very likely that the
-								// dir listing will be stale and will claim a
-								// file segments_X exists when in fact it
-								// doesn't.  So, we catch this and handle it
-								// as if the file does not exist
-								if (infoStream != null)
-								{
-									Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
-								}
-								sis = null;
-							}
-							if (sis != null)
-							{
-								CommitPoint commitPoint = new CommitPoint(this, sis);
-								if (sis.GetGeneration() == segmentInfos.GetGeneration())
-								{
-									currentCommitPoint = commitPoint;
-								}
-								commits.Add(commitPoint);
-								IncRef(sis, true);
-							}
-						}
-					}
-				}
-			}
-			
-			if (currentCommitPoint == null)
-			{
-				// We did not in fact see the segments_N file
-				// corresponding to the segmentInfos that was passed
-				// in.  Yet, it must exist, because our caller holds
-				// the write lock.  This can happen when the directory
-				// listing was stale (eg when index accessed via NFS
-				// client with stale directory listing cache).  So we
-				// try now to explicitly open this commit point:
-				SegmentInfos sis = new SegmentInfos();
-				try
-				{
-					sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
-				}
-				catch (System.IO.IOException e)
-				{
-					throw new CorruptIndexException("failed to locate current segments_N file");
-				}
-				if (infoStream != null)
-					Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
-				currentCommitPoint = new CommitPoint(this, sis);
-				commits.Add(currentCommitPoint);
-				IncRef(sis, true);
-			}
-			
-			// We keep commits list in sorted order (oldest to newest):
-			commits.Sort();
-			
-			// Now delete anything with ref count at 0.  These are
-			// presumably abandoned files eg due to crash of
-			// IndexWriter.
-			System.Collections.IEnumerator it = refCounts.Keys.GetEnumerator();
-			while (it.MoveNext())
-			{
-				System.String fileName = (System.String) it.Current;
-				RefCount rc = (RefCount) refCounts[fileName];
-				if (0 == rc.count)
-				{
-					if (infoStream != null)
-					{
-						Message("init: removing unreferenced file \"" + fileName + "\"");
-					}
-					DeleteFile(fileName);
-				}
-			}
-			
-			// Finally, give policy a chance to remove things on
-			// startup:
-			policy.OnInit(commits);
-			
-			// It's OK for the onInit to remove the current commit
-			// point; we just have to checkpoint our in-memory
-			// SegmentInfos to protect those files that it uses:
-			if (currentCommitPoint.deleted)
-			{
-				Checkpoint(segmentInfos, false);
-			}
-			
-			DeleteCommits();
-		}
-		
-		/// <summary> Remove the CommitPoints in the commitsToDelete List by
-		/// DecRef'ing all files from each SegmentInfos.
-		/// </summary>
-		private void  DeleteCommits()
-		{
-			
-			int size = commitsToDelete.Count;
-			
-			if (size > 0)
-			{
-				
-				// First decref all files that had been referred to by
-				// the now-deleted commits:
-				for (int i = 0; i < size; i++)
-				{
-					CommitPoint commit = (CommitPoint) commitsToDelete[i];
-					if (infoStream != null)
-					{
-						Message("deleteCommits: now remove commit \"" + commit.GetSegmentsFileName() + "\"");
-					}
-					int size2 = commit.files.Count;
-					for (int j = 0; j < size2; j++)
-					{
-						DecRef((System.String) commit.files[j]);
-					}
-				}
-				commitsToDelete.Clear();
-				
-				// Now compact commits to remove deleted ones (preserving the sort):
-				size = commits.Count;
-				int readFrom = 0;
-				int writeTo = 0;
-				while (readFrom < size)
-				{
-					CommitPoint commit = (CommitPoint) commits[readFrom];
-					if (!commit.deleted)
-					{
-						if (writeTo != readFrom)
-						{
-							commits[writeTo] = commits[readFrom];
-						}
-						writeTo++;
-					}
-					readFrom++;
-				}
-				
-				while (size > writeTo)
-				{
-					commits.RemoveAt(size - 1);
-					size--;
-				}
-			}
-		}
-		
-		/// <summary> Writer calls this when it has hit an error and had to
-		/// roll back, to tell us that there may now be
-		/// unreferenced files in the filesystem.  So we re-list
-		/// the filesystem and delete such files.  If segmentName
-		/// is non-null, we will only delete files corresponding to
-		/// that segment.
-		/// </summary>
-		public void  Refresh(System.String segmentName)
-		{
-			System.String[] files = directory.List();
-			if (files == null)
-			{
-				throw new System.IO.IOException("cannot read directory " + directory + ": list() returned null");
-			}
-			IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
-			System.String segmentPrefix1;
-			System.String segmentPrefix2;
-			if (segmentName != null)
-			{
-				segmentPrefix1 = segmentName + ".";
-				segmentPrefix2 = segmentName + "_";
-			}
-			else
-			{
-				segmentPrefix1 = null;
-				segmentPrefix2 = null;
-			}
-			
-			for (int i = 0; i < files.Length; i++)
-			{
-				System.String fileName = files[i];
-				if (filter.Accept(null, fileName) && (segmentName == null || fileName.StartsWith(segmentPrefix1) || fileName.StartsWith(segmentPrefix2)) && !refCounts.Contains(fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
-				{
-					// Unreferenced file, so remove it
-					if (infoStream != null)
-					{
-						Message("refresh [prefix=" + segmentName + "]: removing newly created unreferenced file \"" + fileName + "\"");
-					}
-					DeleteFile(fileName);
-				}
-			}
-		}
-		
-		public void  Refresh()
-		{
-			Refresh(null);
-		}
-		
-		public void  Close()
-		{
-			DeletePendingFiles();
-		}
-		
-		private void  DeletePendingFiles()
-		{
-			if (deletable != null)
-			{
-				System.Collections.IList oldDeletable = deletable;
-				deletable = null;
-				int size = oldDeletable.Count;
-				for (int i = 0; i < size; i++)
-				{
-					if (infoStream != null)
-					{
-						Message("delete pending file " + oldDeletable[i]);
-					}
-					DeleteFile((System.String) oldDeletable[i]);
-				}
-			}
-		}
-		
-		/// <summary> For definition of "check point" see IndexWriter comments:
-		/// "Clarification: Check Points (and commits)".
-		/// 
-		/// Writer calls this when it has made a "consistent
-		/// change" to the index, meaning new files are written to
-		/// the index and the in-memory SegmentInfos have been
-		/// modified to point to those files.
-		/// 
-		/// This may or may not be a commit (segments_N may or may
-		/// not have been written).
-		/// 
-		/// We simply incref the files referenced by the new
-		/// SegmentInfos and decref the files we had previously
-		/// seen (if any).
-		/// 
-		/// If this is a commit, we also call the policy to give it
-		/// a chance to remove other commits.  If any commits are
-		/// removed, we decref their files as well.
-		/// </summary>
-		public void  Checkpoint(SegmentInfos segmentInfos, bool isCommit)
-		{
-			
-			if (infoStream != null)
-			{
-				Message("now checkpoint \"" + segmentInfos.GetCurrentSegmentFileName() + "\" [" + segmentInfos.Count + " segments " + "; isCommit = " + isCommit + "]");
-			}
-			
-			// Try again now to delete any previously un-deletable
-			// files (because they were in use, on Windows):
-			DeletePendingFiles();
-			
-			// Incref the files:
-			IncRef(segmentInfos, isCommit);
-			System.Collections.IList docWriterFiles;
-			if (docWriter != null)
-			{
-				docWriterFiles = docWriter.Files();
-				if (docWriterFiles != null)
-					IncRef(docWriterFiles);
-			}
-			else
-				docWriterFiles = null;
-			
-			if (isCommit)
-			{
-				// Append to our commits list:
-				commits.Add(new CommitPoint(this, segmentInfos));
-				
-				// Tell policy so it can remove commits:
-				policy.OnCommit(commits);
-				
-				// Decref files for commits that were deleted by the policy:
-				DeleteCommits();
-			}
-			
-			// DecRef old files from the last checkpoint, if any:
-			int size = lastFiles.Count;
-			if (size > 0)
-			{
-				for (int i = 0; i < size; i++)
-					DecRef((System.Collections.IList) lastFiles[i]);
-				lastFiles.Clear();
-			}
-			
-			if (!isCommit)
-			{
-				// Save files so we can decr on next checkpoint/commit:
-				size = segmentInfos.Count;
-				for (int i = 0; i < size; i++)
-				{
-					SegmentInfo segmentInfo = segmentInfos.Info(i);
-					if (segmentInfo.dir == directory)
-					{
-						lastFiles.Add(segmentInfo.Files());
-					}
-				}
-			}
-			if (docWriterFiles != null)
-				lastFiles.Add(docWriterFiles);
-		}
-		
-		internal void  IncRef(SegmentInfos segmentInfos, bool isCommit)
-		{
-			int size = segmentInfos.Count;
-			for (int i = 0; i < size; i++)
-			{
-				SegmentInfo segmentInfo = segmentInfos.Info(i);
-				if (segmentInfo.dir == directory)
-				{
-					IncRef(segmentInfo.Files());
-				}
-			}
-			
-			if (isCommit)
-			{
-				// Since this is a commit point, also incref its
-				// segments_N file:
-				GetRefCount(segmentInfos.GetCurrentSegmentFileName()).IncRef();
-			}
-		}
-		
-		internal void  IncRef(System.Collections.IList files)
-		{
-			int size = files.Count;
-			for (int i = 0; i < size; i++)
-			{
-				System.String fileName = (System.String) files[i];
-				RefCount rc = GetRefCount(fileName);
-				if (infoStream != null && VERBOSE_REF_COUNTS)
-				{
-					Message("  IncRef \"" + fileName + "\": pre-incr count is " + rc.count);
-				}
-				rc.IncRef();
-			}
-		}
-		
-		internal void  DecRef(System.Collections.IList files)
-		{
-			int size = files.Count;
-			for (int i = 0; i < size; i++)
-			{
-				DecRef((System.String) files[i]);
-			}
-		}
-		
-		private void  DecRef(System.String fileName)
-		{
-			RefCount rc = GetRefCount(fileName);
-			if (infoStream != null && VERBOSE_REF_COUNTS)
-			{
-				Message("  DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
-			}
-			if (0 == rc.DecRef())
-			{
-				// This file is no longer referenced by any past
-				// commit points nor by the in-memory SegmentInfos:
-				DeleteFile(fileName);
-				refCounts.Remove(fileName);
-			}
-		}
-		
-		internal void  DecRef(SegmentInfos segmentInfos)
-		{
-			int size = segmentInfos.Count;
-			for (int i = 0; i < size; i++)
-			{
-				SegmentInfo segmentInfo = segmentInfos.Info(i);
-				if (segmentInfo.dir == directory)
-				{
-					DecRef(segmentInfo.Files());
-				}
-			}
-		}
-		
-		private RefCount GetRefCount(System.String fileName)
-		{
-			RefCount rc;
-			if (!refCounts.Contains(fileName))
-			{
-				rc = new RefCount();
-				refCounts[fileName] = rc;
-			}
-			else
-			{
-				rc = (RefCount) refCounts[fileName];
-			}
-			return rc;
-		}
-		
-		internal void  DeleteFiles(System.Collections.IList files)
-		{
-			int size = files.Count;
-			for (int i = 0; i < size; i++)
-				DeleteFile((System.String) files[i]);
-		}
-		
-		/// <summary>Delets the specified files, but only if they are new
-		/// (have not yet been incref'd). 
-		/// </summary>
-		internal void  DeleteNewFiles(System.Collections.IList files)
-		{
-			int size = files.Count;
-			for (int i = 0; i < size; i++)
-				if (!refCounts.Contains(files[i]))
-					DeleteFile((System.String) files[i]);
-		}
-		
-		internal void  DeleteFile(System.String fileName)
-		{
-			try
-			{
-				if (infoStream != null)
-				{
-					Message("delete \"" + fileName + "\"");
-				}
-				directory.DeleteFile(fileName);
-			}
-			catch (System.IO.IOException e)
-			{
-				// if delete fails
-				if (directory.FileExists(fileName))
-				{
-					
-					// Some operating systems (e.g. Windows) don't
-					// permit a file to be deleted while it is opened
-					// for read (e.g. by another process or thread). So
-					// we assume that when a delete fails it is because
-					// the file is open in another process, and queue
-					// the file for subsequent deletion.
-					
-					if (infoStream != null)
-					{
-						Message("IndexFileDeleter: unable to remove file \"" + fileName + "\": " + e.ToString() + "; Will re-try later.");
-					}
-					if (deletable == null)
-					{
-						deletable = new System.Collections.ArrayList();
-					}
-					deletable.Add(fileName); // add to deletable
-				}
-			}
-		}
-		
-		/// <summary> Tracks the reference count for a single index file:</summary>
-		sealed private class RefCount
-		{
-			
-			internal int count;
-			
-			public int IncRef()
-			{
-				return ++count;
-			}
-			
-			public int DecRef()
-			{
-				System.Diagnostics.Debug.Assert(count > 0);
-				return --count;
-			}
-		}
-		
-		/// <summary> Holds details for each commit point.  This class is
-		/// also passed to the deletion policy.  Note: this class
-		/// has a natural ordering that is inconsistent with
-		/// equals.
-		/// </summary>
-		
-		sealed private class CommitPoint : System.IComparable, IndexCommitPoint
-		{
-			private void  InitBlock(IndexFileDeleter enclosingInstance)
-			{
-				this.enclosingInstance = enclosingInstance;
-			}
-			private IndexFileDeleter enclosingInstance;
-			public IndexFileDeleter Enclosing_Instance
-			{
-				get
-				{
-					return enclosingInstance;
-				}
-				
-			}
-			
-			internal long gen;
-			internal System.Collections.IList files;
-			internal System.String segmentsFileName;
-			internal bool deleted;
-			
-			public CommitPoint(IndexFileDeleter enclosingInstance, SegmentInfos segmentInfos)
-			{
-				InitBlock(enclosingInstance);
-				segmentsFileName = segmentInfos.GetCurrentSegmentFileName();
-				int size = segmentInfos.Count;
-				files = new System.Collections.ArrayList(size);
-				files.Add(segmentsFileName);
-				gen = segmentInfos.GetGeneration();
-				for (int i = 0; i < size; i++)
-				{
-					SegmentInfo segmentInfo = segmentInfos.Info(i);
-					if (segmentInfo.dir == Enclosing_Instance.directory)
-					{
-                        System.Collections.IEnumerator filesEnum = segmentInfo.Files().GetEnumerator();
-                        while (filesEnum.MoveNext())
+        private Directory directory;
+        private IndexDeletionPolicy policy;
+        private DocumentsWriter docWriter;
+
+        /// <summary>Change to true to see details of reference counts when
+        /// infoStream != null 
+        /// </summary>
+        public static bool VERBOSE_REF_COUNTS = false;
+
+        internal void SetInfoStream(System.IO.TextWriter infoStream)
+        {
+            this.infoStream = infoStream;
+            if (infoStream != null)
+            {
+                Message("setInfoStream deletionPolicy=" + policy);
+            }
+        }
+
+        private void Message(string message)
+        {
+            infoStream.WriteLine("IFD [" + SupportClass.ThreadClass.Current().Name + "]: " + message);
+        }
+
+        /// <summary> Initialize the deleter: find all previous commits in
+        /// the Directory, incref the files they reference, call
+        /// the policy to let it delete commits.  The incoming
+        /// segmentInfos must have been loaded from a commit point
+        /// and not yet modified.  This will remove any files not
+        /// referenced by any of the commits.
+        /// </summary>
+        /// <throws>  CorruptIndexException if the index is corrupt </throws>
+        /// <throws>  IOException if there is a low-level IO error </throws>
+        public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.TextWriter infoStream, DocumentsWriter docWriter)
+        {
+
+            this.docWriter = docWriter;
+            this.infoStream = infoStream;
+
+            if (infoStream != null)
+            {
+                Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
+            }
+
+            this.policy = policy;
+            this.directory = directory;
+
+            // First pass: walk the files and initialize our ref
+            // counts:
+            long currentGen = segmentInfos.GetGeneration();
+            IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
+
+            string[] files = directory.List();
+            if (files == null)
+            {
+                throw new System.IO.IOException("cannot read directory " + directory + ": list() returned null");
+            }
+
+            CommitPoint currentCommitPoint = null;
+
+            for (int i = 0; i < files.Length; i++)
+            {
+
+                string fileName = files[i];
+
+                if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
+                {
+
+                    // Add this file to refCounts with initial count 0:
+                    GetRefCount(fileName);
+
+                    if (fileName.StartsWith(IndexFileNames.SEGMENTS))
+                    {
+
+                        // This is a commit (segments or segments_N), and
+                        // it's valid (<= the max gen).  Load it, then
+                        // incref all files it refers to:
+                        if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
                         {
-                            files.Add(filesEnum.Current);
+                            if (infoStream != null)
+                            {
+                                Message("init: load commit \"" + fileName + "\"");
+                            }
+                            SegmentInfos sis = new SegmentInfos();
+                            try
+                            {
+                                sis.Read(directory, fileName);
+                            }
+                            catch (System.IO.FileNotFoundException)
+                            {
+                                // LUCENE-948: on NFS (and maybe others), if
+                                // you have writers switching back and forth
+                                // between machines, it's very likely that the
+                                // dir listing will be stale and will claim a
+                                // file segments_X exists when in fact it
+                                // doesn't.  So, we catch this and handle it
+                                // as if the file does not exist
+                                if (infoStream != null)
+                                {
+                                    Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
+                                }
+                                sis = null;
+                            }
+                            if (sis != null)
+                            {
+                                CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
+                                if (sis.GetGeneration() == segmentInfos.GetGeneration())
+                                {
+                                    currentCommitPoint = commitPoint;
+                                }
+                                commits.Add(commitPoint);
+                                IncRef(sis, true);
+                            }
                         }
-					}
-				}
-			}
-			
-			/// <summary> Get the segments_N file for this commit point.</summary>
-			public System.String GetSegmentsFileName()
-			{
-				return segmentsFileName;
-			}
-			
-			public System.Collections.ICollection GetFileNames()
-			{
-				return System.Collections.ArrayList.ReadOnly(new System.Collections.ArrayList(files));
-			}
-			
-			/// <summary> Called only be the deletion policy, to remove this
-			/// commit point from the index.
-			/// </summary>
-			public void  Delete()
-			{
-				if (!deleted)
-				{
-					deleted = true;
-					Enclosing_Instance.commitsToDelete.Add(this);
-				}
-			}
-			
-			public int CompareTo(System.Object obj)
-			{
-				CommitPoint commit = (CommitPoint) obj;
-				if (gen < commit.gen)
-				{
-					return - 1;
-				}
-				else if (gen > commit.gen)
-				{
-					return 1;
-				}
-				else
-				{
-					return 0;
-				}
-			}
-		}
-	}
+                    }
+                }
+            }
+
+            if (currentCommitPoint == null)
+            {
+                // We did not in fact see the segments_N file
+                // corresponding to the segmentInfos that was passed
+                // in.  Yet, it must exist, because our caller holds
+                // the write lock.  This can happen when the directory
+                // listing was stale (eg when index accessed via NFS
+                // client with stale directory listing cache).  So we
+                // try now to explicitly open this commit point:
+                SegmentInfos sis = new SegmentInfos();
+                try
+                {
+                    sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
+                }
+                catch (System.IO.IOException)
+                {
+                    throw new CorruptIndexException("failed to locate current segments_N file");
+                }
+                if (infoStream != null)
+                    Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
+                currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
+                commits.Add(currentCommitPoint);
+                IncRef(sis, true);
+            }
+
+            // We keep commits list in sorted order (oldest to newest):
+            commits.Sort();
+
+            // Now delete anything with ref count at 0.  These are
+            // presumably abandoned files eg due to crash of
+            // IndexWriter.
+            IEnumerator<KeyValuePair<string,RefCount>> it = refCounts.GetEnumerator();
+            while (it.MoveNext())
+            {
+                string fileName = it.Current.Key;
+                RefCount rc = it.Current.Value;
+                if (0 == rc.count)
+                {
+                    if (infoStream != null)
+                    {
+                        Message("init: removing unreferenced file \"" + fileName + "\"");
+                    }
+                    DeleteFile(fileName);
+                }
+            }
+
+            // Finally, give policy a chance to remove things on
+            // startup:
+            policy.OnInit(commits);
+
+            // It's OK for the onInit to remove the current commit
+            // point; we just have to checkpoint our in-memory
+            // SegmentInfos to protect those files that it uses:
+            if (currentCommitPoint.deleted)
+            {
+                Checkpoint(segmentInfos, false);
+            }
+
+            DeleteCommits();
+        }
+
+        /// <summary> Remove the CommitPoints in the commitsToDelete List by
+        /// DecRef'ing all files from each SegmentInfos.
+        /// </summary>
+        private void DeleteCommits()
+        {
+
+            int size = commitsToDelete.Count;
+
+            if (size > 0)
+            {
+
+                // First decref all files that had been referred to by
+                // the now-deleted commits:
+                for (int i = 0; i < size; i++)
+                {
+                    CommitPoint commit = (CommitPoint)commitsToDelete[i];
+                    if (infoStream != null)
+                    {
+                        Message("deleteCommits: now decRef commit \"" + commit.GetSegmentsFileName() + "\"");
+                    }
+                    int size2 = commit.files.Count;
+                    for (int j = 0; j < size2; j++)
+                    {
+                        DecRef((string)commit.files[j]);
+                    }
+                }
+                commitsToDelete.Clear();
+
+                // Now compact commits to remove deleted ones (preserving the sort):
+                size = commits.Count;
+                int readFrom = 0;
+                int writeTo = 0;
+                while (readFrom < size)
+                {
+                    CommitPoint commit = (CommitPoint) commits[readFrom];
+                    if (!commit.deleted)
+                    {
+                        if (writeTo != readFrom)
+                        {
+                            commits[writeTo] = commits[readFrom];
+                        }
+                        writeTo++;
+                    }
+                    readFrom++;
+                }
+
+                while (size > writeTo)
+                {
+                    commits.RemoveAt(size - 1);
+                    size--;
+                }
+            }
+        }
+
+        /// <summary> Writer calls this when it has hit an error and had to
+        /// roll back, to tell us that there may now be
+        /// unreferenced files in the filesystem.  So we re-list
+        /// the filesystem and delete such files.  If segmentName
+        /// is non-null, we will only delete files corresponding to
+        /// that segment.
+        /// </summary>
+        public void Refresh(string segmentName)
+        {
+            string[] files = directory.List();
+            if (files == null)
+            {
+                throw new System.IO.IOException("cannot read directory " + directory + ": list() returned null");
+            }
+            IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
+            string segmentPrefix1;
+            string segmentPrefix2;
+            if (segmentName != null)
+            {
+                segmentPrefix1 = segmentName + ".";
+                segmentPrefix2 = segmentName + "_";
+            }
+            else
+            {
+                segmentPrefix1 = null;
+                segmentPrefix2 = null;
+            }
+
+            for (int i = 0; i < files.Length; i++)
+            {
+                string fileName = files[i];
+                if (filter.Accept(null, fileName) && (segmentName == null || fileName.StartsWith(segmentPrefix1) || fileName.StartsWith(segmentPrefix2)) && !refCounts.ContainsKey(fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
+                {
+                    // Unreferenced file, so remove it
+                    if (infoStream != null)
+                    {
+                        Message("refresh [prefix=" + segmentName + "]: removing newly created unreferenced file \"" + fileName + "\"");
+                    }
+                    DeleteFile(fileName);
+                }
+            }
+        }
+
+        public void Refresh()
+        {
+            Refresh(null);
+        }
+
+        public void Close()
+        {
+            DeletePendingFiles();
+        }
+
+        private void DeletePendingFiles()
+        {
+            if (deletable != null)
+            {
+                System.Collections.IList oldDeletable = deletable;
+                deletable = null;
+                int size = oldDeletable.Count;
+                for (int i = 0; i < size; i++)
+                {
+                    if (infoStream != null)
+                    {
+                        Message("delete pending file " + oldDeletable[i]);
+                    }
+                    DeleteFile((string)oldDeletable[i]);
+                }
+            }
+        }
+
+        /// <summary> For definition of "check point" see IndexWriter comments:
+        /// "Clarification: Check Points (and commits)".
+        /// 
+        /// Writer calls this when it has made a "consistent
+        /// change" to the index, meaning new files are written to
+        /// the index and the in-memory SegmentInfos have been
+        /// modified to point to those files.
+        /// 
+        /// This may or may not be a commit (segments_N may or may
+        /// not have been written).
+        /// 
+        /// We simply incref the files referenced by the new
+        /// SegmentInfos and decref the files we had previously
+        /// seen (if any).
+        /// 
+        /// If this is a commit, we also call the policy to give it
+        /// a chance to remove other commits.  If any commits are
+        /// removed, we decref their files as well.
+        /// </summary>
+        public void Checkpoint(SegmentInfos segmentInfos, bool isCommit)
+        {
+
+            if (infoStream != null)
+            {
+                Message("now checkpoint \"" + segmentInfos.GetCurrentSegmentFileName() + "\" [" + segmentInfos.Count + " segments " + "; isCommit = " + isCommit + "]");
+            }
+
+            // Try again now to delete any previously un-deletable
+            // files (because they were in use, on Windows):
+            DeletePendingFiles();
+
+            // Incref the files:
+            IncRef(segmentInfos, isCommit);
+
+            if (isCommit)
+            {
+                // append to our commits list
+                commits.Add(new CommitPoint(this, commitsToDelete, directory, segmentInfos));
+
+                // tell policy so it can remove commits
+                policy.OnCommit(commits);
+
+                // decref files for commits that were deleted by the policy
+                DeleteCommits();
+            }
+            else
+            {
+
+                List<string> docWriterFiles;
+                if (docWriter != null)
+                {
+                    docWriterFiles = docWriter.OpenFiles();
+                    if (docWriterFiles != null)
+                        // we must incRef these files before decRef'ing
+                        // last files to make sure we don't accidentally 
+                        // delete them
+                        IncRef(docWriterFiles);
+                }
+                else
+                    docWriterFiles = null;
+
+                // DecRef old files from the last checkpoint, if any:
+                int size = lastFiles.Count;
+                if (size > 0)
+                {
+                    for (int i = 0; i < size; i++)
+                        DecRef(lastFiles[i]);
+                    lastFiles.Clear();
+                }
+
+                // Save files so we can decr on next checkpoint/commit:
+                size = segmentInfos.Count;
+                for (int i = 0; i < size; i++)
+                {
+                    SegmentInfo segmentInfo = segmentInfos.Info(i);
+                    if (segmentInfo.dir == directory)
+                    {
+                        SupportClass.CollectionsSupport.AddAll(segmentInfo.Files(), lastFiles);
+                    }
+                }
+
+                if (docWriterFiles != null)
+                    SupportClass.CollectionsSupport.AddAll(docWriterFiles, lastFiles);
+            }
+        }
+
+        internal void IncRef(SegmentInfos segmentInfos, bool isCommit)
+        {
+            int size = segmentInfos.Count;
+            for (int i = 0; i < size; i++)
+            {
+                SegmentInfo segmentInfo = segmentInfos.Info(i);
+                if (segmentInfo.dir == directory)
+                {
+                    IncRef(segmentInfo.Files());
+                }
+            }
+
+            if (isCommit)
+            {
+                // Since this is a commit point, also incref its
+                // segments_N file:
+                GetRefCount(segmentInfos.GetCurrentSegmentFileName()).IncRef();
+            }
+        }
+
+        internal void IncRef(System.Collections.Generic.IList<string> files)
+        {
+            int size = files.Count;
+            for (int i = 0; i < size; i++)
+            {
+                string fileName = files[i];
+                RefCount rc = GetRefCount(fileName);
+                if (infoStream != null && VERBOSE_REF_COUNTS)
+                {
+                    Message("  IncRef \"" + fileName + "\": pre-incr count is " + rc.count);
+                }
+                rc.IncRef();
+            }
+        }
+
+        internal void DecRef(System.Collections.Generic.IList<string> files)
+        {
+            int size = files.Count;
+            for (int i = 0; i < size; i++)
+            {
+                DecRef(files[i]);
+            }
+        }
+
+        internal void DecRef(string fileName)
+        {
+            RefCount rc = GetRefCount(fileName);
+            if (infoStream != null && VERBOSE_REF_COUNTS)
+            {
+                Message("  DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
+            }
+            if (0 == rc.DecRef())
+            {
+                // This file is no longer referenced by any past
+                // commit points nor by the in-memory SegmentInfos:
+                DeleteFile(fileName);
+                refCounts.Remove(fileName);
+            }
+        }
+
+        internal void DecRef(SegmentInfos segmentInfos)
+        {
+            int size = segmentInfos.Count;
+            for (int i = 0; i < size; i++)
+            {
+                SegmentInfo segmentInfo = segmentInfos.Info(i);
+                if (segmentInfo.dir == directory)
+                {
+                    DecRef(segmentInfo.Files());
+                }
+            }
+        }
+
+        private RefCount GetRefCount(string fileName)
+        {
+            RefCount rc;
+            if (!refCounts.ContainsKey(fileName))
+            {
+                rc = new RefCount();
+                refCounts[fileName] = rc;
+            }
+            else
+            {
+                rc = refCounts[fileName];
+            }
+            return rc;
+        }
+
+        internal void DeleteFiles(List<string> files)
+        {
+            int size = files.Count;
+            for (int i = 0; i < size; i++)
+                DeleteFile(files[i]);
+        }
+
+        /// <summary>Delets the specified files, but only if they are new
+        /// (have not yet been incref'd). 
+        /// </summary>
+        internal void DeleteNewFiles(ICollection<string> files)
+        {
+            IEnumerator<string> enumerator = files.GetEnumerator();
+            while (enumerator.MoveNext())
+            {
+                string fileName = enumerator.Current;
+                if (!refCounts.ContainsKey(fileName))
+                    DeleteFile(fileName);
+            }
+        }
+
+        internal void DeleteFile(string fileName)
+        {
+            try
+            {
+                if (infoStream != null)
+                {
+                    Message("delete \"" + fileName + "\"");
+                }
+                directory.DeleteFile(fileName);
+            }
+            catch (System.IO.IOException e)
+            {
+                // if delete fails
+                if (directory.FileExists(fileName))
+                {
+
+                    // Some operating systems (e.g. Windows) don't
+                    // permit a file to be deleted while it is opened
+                    // for read (e.g. by another process or thread). So
+                    // we assume that when a delete fails it is because
+                    // the file is open in another process, and queue
+                    // the file for subsequent deletion.
+
+                    if (infoStream != null)
+                    {
+                        Message("IndexFileDeleter: unable to remove file \"" + fileName + "\": " + e.ToString() + "; Will re-try later.");
+                    }
+                    if (deletable == null)
+                    {
+                        deletable = new List<string>();
+                    }
+                    deletable.Add(fileName); // add to deletable
+                }
+            }
+        }
+
+        /// <summary> Tracks the reference count for a single index file:</summary>
+        sealed private class RefCount
+        {
+
+            internal int count;
+
+            public int IncRef()
+            {
+                return ++count;
+            }
+
+            public int DecRef()
+            {
+                System.Diagnostics.Debug.Assert(count > 0);
+                return --count;
+            }
+        }
+
+        /// <summary> Holds details for each commit point.  This class is
+        /// also passed to the deletion policy.  Note: this class
+        /// has a natural ordering that is inconsistent with
+        /// equals.
+        /// </summary>
+
+        sealed private class CommitPoint : IndexCommit, System.IComparable
+        {
+            private void InitBlock(IndexFileDeleter enclosingInstance)
+            {
+                this.enclosingInstance = enclosingInstance;
+            }
+            private IndexFileDeleter enclosingInstance;
+            public IndexFileDeleter Enclosing_Instance
+            {
+                get
+                {
+                    return enclosingInstance;
+                }
+
+            }
+
+            internal long gen;
+            internal List<string> files;
+            internal string segmentsFileName;
+            internal bool deleted;
+            internal Directory directory;
+            internal ICollection<IndexCommitPoint> commitsToDelete;
+            internal long version;
+            internal long generation;
+            internal readonly bool isOptimized;
+
+            public CommitPoint(IndexFileDeleter enclosingInstance, ICollection<IndexCommitPoint> commitsToDelete, Directory directory, SegmentInfos segmentInfos)
+            {
+                InitBlock(enclosingInstance);
+                this.directory = directory;
+                this.commitsToDelete = commitsToDelete;
+                segmentsFileName = segmentInfos.GetCurrentSegmentFileName();
+                version = segmentInfos.GetVersion();
+                generation = segmentInfos.GetGeneration();
+                int size = segmentInfos.Count;
+                files = new List<string>(size);
+                files.Add(segmentsFileName);
+                gen = segmentInfos.GetGeneration();
+                for (int i = 0; i < size; i++)
+                {
+                    SegmentInfo segmentInfo = segmentInfos.Info(i);
+                    if (segmentInfo.dir == Enclosing_Instance.directory)
+                    {
+                        SupportClass.CollectionsSupport.AddAll(segmentInfo.Files(), files);
+                    }
+                }
+                isOptimized = segmentInfos.Count == 1 && !segmentInfos.Info(0).HasDeletions();
+            }
+
+            public override bool IsOptimized()
+            {
+                return isOptimized;
+            }
+
+            public override string GetSegmentsFileName()
+            {
+                return segmentsFileName;
+            }
+
+            public override ICollection<string> GetFileNames()
+            {
+                return files.AsReadOnly();
+            }
+
+            public override Directory GetDirectory()
+            {
+                return directory;
+            }
+
+            public override long GetVersion()
+            {
+                return version;
+            }
+
+            public override long GetGeneration()
+            {
+                return generation;
+            }
+
+            /// <summary> Called only be the deletion policy, to remove this
+            /// commit point from the index.
+            /// </summary>
+            public override void Delete()
+            {
+                if (!deleted)
+                {
+                    deleted = true;
+                    Enclosing_Instance.commitsToDelete.Add(this);
+                }
+            }
+
+            public override bool IsDeleted()
+            {
+                return deleted;
+            }
+
+            public int CompareTo(object obj)
+            {
+                CommitPoint commit = (CommitPoint)obj;
+                if (gen < commit.gen)
+                {
+                    return -1;
+                }
+                else if (gen > commit.gen)
+                {
+                    return 1;
+                }
+                else
+                {
+                    return 0;
+                }
+            }
+        }
+    }
 }
\ No newline at end of file



Mime
View raw message