lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aro...@apache.org
Subject svn commit: r671404 [1/10] - /incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/
Date Wed, 25 Jun 2008 02:52:24 GMT
Author: aroush
Date: Tue Jun 24 19:52:22 2008
New Revision: 671404

URL: http://svn.apache.org/viewvc?rev=671404&view=rev
Log:
Release: Apache Lucene.Net.2.3.1 build 001 "Alpha"

Added:
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CorruptIndexException.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldSortedTermVectorMapper.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommitPoint.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexDeletionPolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/KeepOnlyLastCommitDeletionPolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogByteSizeMergePolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogDocMergePolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogMergePolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergePolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergeScheduler.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiLevelSkipListReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiLevelSkipListWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiSegmentReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Payload.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/PositionBasedTermVectorMapper.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReade-2r.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SerialMergeScheduler.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SnapshotDeletionPolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SortedTermVectorMapper.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StaleReaderException.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorEntry.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorEntryFreqSortedComparator.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorMapper.cs
Modified:
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldInfo.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldInfos.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldReaderException.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FilterIndexReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileNameFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileNames.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexModifier.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultipleTermPositions.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Package.html
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfo.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMergeQueue.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermDocs.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermEnum.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermPositionVector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermPositions.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermVector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Term.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermBuffer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermFreqVector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermInfosReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermInfosWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermPositions.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorOffsetInfo.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsWriter.cs

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/CheckIndex.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,429 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using Document = Lucene.Net.Documents.Document;
+using Directory = Lucene.Net.Store.Directory;
+using FSDirectory = Lucene.Net.Store.FSDirectory;
+using IndexInput = Lucene.Net.Store.IndexInput;
+
+namespace Lucene.Net.Index
+{
+	
+	/// <summary> Basic tool to check the health of an index and write a
+	/// new segments file that removes reference to problematic
+	/// segments.  There are many more checks that this tool
+	/// could do but does not yet, eg: reconstructing a segments
+	/// file by looking for all loadable segments (if no segments
+	/// file is found), removing specifically specified segments,
+	/// listing files that exist but are not referenced, etc.
+	/// </summary>
+	
+	public class CheckIndex
+	{
+
+        public static System.IO.TextWriter out_Renamed;
+		
+		private class MySegmentTermDocs : SegmentTermDocs
+		{
+			
+			internal int delCount;
+			
+			internal MySegmentTermDocs(SegmentReader p) : base(p)
+			{
+			}
+			
+			public override void  Seek(Term term)
+			{
+				base.Seek(term);
+				delCount = 0;
+			}
+			
+			protected internal override void  SkippingDoc()
+			{
+				delCount++;
+			}
+		}
+		
+		/// <summary>Returns true if index is clean, else false.</summary>
+		public static bool Check(Directory dir, bool doFix)
+		{
+			System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat;
+			SegmentInfos sis = new SegmentInfos();
+			
+			try
+			{
+				sis.Read(dir);
+			}
+			catch (System.Exception t)
+			{
+				out_Renamed.WriteLine("ERROR: could not read any segments file in directory");
+				out_Renamed.Write(t.StackTrace);
+				out_Renamed.Flush();
+				return false;
+			}
+			
+			int numSegments = sis.Count;
+			System.String segmentsFileName = sis.GetCurrentSegmentFileName();
+			IndexInput input = null;
+			try
+			{
+				input = dir.OpenInput(segmentsFileName);
+			}
+			catch (System.Exception t)
+			{
+				out_Renamed.WriteLine("ERROR: could not open segments file in directory");
+				out_Renamed.Write(t.StackTrace);
+				out_Renamed.Flush();
+				return false;
+			}
+			int format = 0;
+			try
+			{
+				format = input.ReadInt();
+			}
+			catch (System.Exception t)
+			{
+				out_Renamed.WriteLine("ERROR: could not read segment file version in directory");
+				out_Renamed.Write(t.StackTrace);
+				out_Renamed.Flush();
+				return false;
+			}
+			finally
+			{
+				if (input != null)
+					input.Close();
+			}
+			
+			System.String sFormat = "";
+			bool skip = false;
+			
+			if (format == SegmentInfos.FORMAT)
+				sFormat = "FORMAT [Lucene Pre-2.1]";
+			if (format == SegmentInfos.FORMAT_LOCKLESS)
+				sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
+			else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
+				sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
+			else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
+				sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
+			else if (format < SegmentInfos.FORMAT_SHARED_DOC_STORE)
+			{
+				sFormat = "int=" + format + " [newer version of Lucene than this tool]";
+				skip = true;
+			}
+			else
+			{
+				sFormat = format + " [Lucene 1.3 or prior]";
+			}
+			
+			out_Renamed.WriteLine("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat);
+			
+			if (skip)
+			{
+				out_Renamed.WriteLine("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
+				return false;
+			}
+			
+			SegmentInfos newSIS = (SegmentInfos) sis.Clone();
+			newSIS.Clear();
+			bool changed = false;
+			int totLoseDocCount = 0;
+			int numBadSegments = 0;
+			for (int i = 0; i < numSegments; i++)
+			{
+				SegmentInfo info = sis.Info(i);
+				out_Renamed.WriteLine("  " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
+				int toLoseDocCount = info.docCount;
+				
+				SegmentReader reader = null;
+				
+				try
+				{
+					out_Renamed.WriteLine("    compound=" + info.GetUseCompoundFile());
+					out_Renamed.WriteLine("    numFiles=" + info.Files().Count);
+					out_Renamed.WriteLine(String.Format(nf, "    size (MB)={0:f}", new Object[] { (info.SizeInBytes() / (1024.0 * 1024.0)) }));
+					int docStoreOffset = info.GetDocStoreOffset();
+					if (docStoreOffset != - 1)
+					{
+						out_Renamed.WriteLine("    docStoreOffset=" + docStoreOffset);
+						out_Renamed.WriteLine("    docStoreSegment=" + info.GetDocStoreSegment());
+						out_Renamed.WriteLine("    docStoreIsCompoundFile=" + info.GetDocStoreIsCompoundFile());
+					}
+					System.String delFileName = info.GetDelFileName();
+					if (delFileName == null)
+						out_Renamed.WriteLine("    no deletions");
+					else
+						out_Renamed.WriteLine("    has deletions [delFileName=" + delFileName + "]");
+					out_Renamed.Write("    test: open reader.........");
+					reader = SegmentReader.Get(info);
+					int numDocs = reader.NumDocs();
+					toLoseDocCount = numDocs;
+					if (reader.HasDeletions())
+						out_Renamed.WriteLine("OK [" + (info.docCount - numDocs) + " deleted docs]");
+					else
+						out_Renamed.WriteLine("OK");
+					
+					out_Renamed.Write("    test: fields, norms.......");
+					System.Collections.ICollection fieldNames = reader.GetFieldNames(IndexReader.FieldOption.ALL);
+					System.Collections.IEnumerator it = fieldNames.GetEnumerator();
+					while (it.MoveNext())
+					{
+						System.String fieldName = (System.String) it.Current;
+						byte[] b = reader.Norms(fieldName);
+						if (b.Length != info.docCount)
+							throw new System.SystemException("norms for field \"" + fieldName + "\" is length " + b.Length + " != maxDoc " + info.docCount);
+					}
+					out_Renamed.WriteLine("OK [" + fieldNames.Count + " fields]");
+					
+					out_Renamed.Write("    test: terms, freq, prox...");
+					TermEnum termEnum = reader.Terms();
+					TermPositions termPositions = reader.TermPositions();
+					
+					// Used only to count up # deleted docs for this
+					// term
+					MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader);
+					
+					long termCount = 0;
+					long totFreq = 0;
+					long totPos = 0;
+					while (termEnum.Next())
+					{
+						termCount++;
+						Term term = termEnum.Term();
+						int docFreq = termEnum.DocFreq();
+						termPositions.Seek(term);
+						int lastDoc = - 1;
+						int freq0 = 0;
+						totFreq += docFreq;
+						while (termPositions.Next())
+						{
+							freq0++;
+							int doc = termPositions.Doc();
+							int freq = termPositions.Freq();
+							if (doc <= lastDoc)
+							{
+								throw new System.SystemException("term " + term + ": doc " + doc + " < lastDoc " + lastDoc);
+							}
+							lastDoc = doc;
+							if (freq <= 0)
+							{
+								throw new System.SystemException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
+							}
+							
+							int lastPos = - 1;
+							totPos += freq;
+							for (int j = 0; j < freq; j++)
+							{
+								int pos = termPositions.NextPosition();
+								if (pos < 0)
+								{
+									throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
+								}
+								if (pos <= lastPos)
+								{
+									throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
+								}
+							}
+						}
+						
+						// Now count how many deleted docs occurred in
+						// this term:
+						int delCount;
+						if (reader.HasDeletions())
+						{
+							myTermDocs.Seek(term);
+							while (myTermDocs.Next())
+							{
+							}
+							delCount = myTermDocs.delCount;
+						}
+						else
+							delCount = 0;
+						
+						if (freq0 + delCount != docFreq)
+						{
+							throw new System.SystemException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
+						}
+					}
+					
+					out_Renamed.WriteLine("OK [" + termCount + " terms; " + totFreq + " terms/docs pairs; " + totPos + " tokens]");
+					
+					out_Renamed.Write("    test: stored fields.......");
+					int docCount = 0;
+					long totFields = 0;
+					for (int j = 0; j < info.docCount; j++)
+						if (!reader.IsDeleted(j))
+						{
+							docCount++;
+							Document doc = reader.Document(j);
+							totFields += doc.GetFields().Count;
+						}
+					
+					if (docCount != reader.NumDocs())
+						throw new System.SystemException("docCount=" + docCount + " but saw " + docCount + " undeleted docs");
+					
+					out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total field count; avg {1:f} fields per doc]", new Object[] { totFields, (((float)totFields) / docCount) }));
+					
+					out_Renamed.Write("    test: term vectors........");
+					int totVectors = 0;
+					for (int j = 0; j < info.docCount; j++)
+						if (!reader.IsDeleted(j))
+						{
+							TermFreqVector[] tfv = reader.GetTermFreqVectors(j);
+							if (tfv != null)
+								totVectors += tfv.Length;
+						}
+					
+					out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total vector count; avg {1:f} term/freq vector fields per doc]", new Object[] { totVectors, (((float)totVectors) / docCount) }));
+					out_Renamed.WriteLine("");
+				}
+				catch (System.Exception t)
+				{
+					out_Renamed.WriteLine("FAILED");
+					System.String comment;
+					if (doFix)
+						comment = "will remove reference to this segment (-fix is specified)";
+					else
+						comment = "would remove reference to this segment (-fix was not specified)";
+					out_Renamed.WriteLine("    WARNING: " + comment + "; full exception:");
+					out_Renamed.Write(t.StackTrace);
+					out_Renamed.Flush();
+					out_Renamed.WriteLine("");
+					totLoseDocCount += toLoseDocCount;
+					numBadSegments++;
+					changed = true;
+					continue;
+				}
+				finally
+				{
+					if (reader != null)
+						reader.Close();
+				}
+				
+				// Keeper
+				newSIS.Add(info.Clone());
+			}
+			
+			if (!changed)
+			{
+				out_Renamed.WriteLine("No problems were detected with this index.\n");
+				return true;
+			}
+			else
+			{
+				out_Renamed.WriteLine("WARNING: " + numBadSegments + " broken segments detected");
+				if (doFix)
+					out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents will be lost");
+				else
+					out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents would be lost if -fix were specified");
+				out_Renamed.WriteLine();
+			}
+			
+			if (doFix)
+			{
+				out_Renamed.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
+				for (int i = 0; i < 5; i++)
+				{
+					try
+					{
+						System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
+					}
+					catch (System.Threading.ThreadInterruptedException)
+					{
+						SupportClass.ThreadClass.Current().Interrupt();
+						i--;
+						continue;
+					}
+					
+					out_Renamed.WriteLine("  " + (5 - i) + "...");
+				}
+				out_Renamed.Write("Writing...");
+				try
+				{
+					newSIS.Write(dir);
+				}
+				catch (System.Exception t)
+				{
+					out_Renamed.WriteLine("FAILED; exiting");
+					out_Renamed.Write(t.StackTrace);
+					out_Renamed.Flush();
+					return false;
+				}
+				out_Renamed.WriteLine("OK");
+				out_Renamed.WriteLine("Wrote new segments file \"" + newSIS.GetCurrentSegmentFileName() + "\"");
+			}
+			else
+			{
+				out_Renamed.WriteLine("NOTE: would write new segments file [-fix was not specified]");
+			}
+			out_Renamed.WriteLine("");
+			
+			return false;
+		}
+		
+		[STAThread]
+		public static void  Main(System.String[] args)
+		{
+			
+			bool doFix = false;
+			for (int i = 0; i < args.Length; i++)
+				if (args[i].Equals("-fix"))
+				{
+					doFix = true;
+					break;
+				}
+			
+			if (args.Length != (doFix ? 2 : 1))
+			{
+				out_Renamed.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix]\n" + "\n" + "  -fix: actually write a new segments_N file, removing any problematic segments\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index.  Always make\n" + "a backup copy of your index before running this!  Do not run this tool on an index\n" + "that is actively being written to.  You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified.  With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file.  This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has has any\n" + "corrup
 tion, else 0.\n");
+				System.Environment.Exit(1);
+			}
+			
+			System.String dirName = args[0];
+			out_Renamed.WriteLine("\nOpening index @ " + dirName + "\n");
+			Directory dir = null;
+			try
+			{
+				dir = FSDirectory.GetDirectory(dirName);
+			}
+			catch (System.Exception t)
+			{
+				out_Renamed.WriteLine("ERROR: could not open directory \"" + dirName + "\"; exiting");
+				out_Renamed.Write(t.StackTrace);
+				out_Renamed.Flush();
+				System.Environment.Exit(1);
+			}
+			
+			bool isClean = Check(dir, doFix);
+			
+			int exitCode;
+			if (isClean)
+				exitCode = 0;
+			else
+				exitCode = 1;
+			System.Environment.Exit(exitCode);
+		}
+		static CheckIndex()
+		{
+			System.IO.StreamWriter temp_writer;
+            temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
+			temp_writer.AutoFlush = true;
+			out_Renamed = temp_writer;
+		}
+	}
+}
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/CompoundFileReader.cs?rev=671404&r1=671403&r2=671404&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs Tue Jun 24 19:52:22 2008
@@ -16,9 +16,10 @@
  */
 
 using System;
+
+using BufferedIndexInput = Lucene.Net.Store.BufferedIndexInput;
 using Directory = Lucene.Net.Store.Directory;
 using IndexInput = Lucene.Net.Store.IndexInput;
-using BufferedIndexInput = Lucene.Net.Store.BufferedIndexInput;
 using IndexOutput = Lucene.Net.Store.IndexOutput;
 using Lock = Lucene.Net.Store.Lock;
 
@@ -30,14 +31,15 @@
 	/// This class implements a directory, but is limited to only read operations.
 	/// Directory methods that would normally modify data throw an exception.
 	/// 
+	/// 
 	/// </summary>
-	/// <author>  Dmitry Serebrennikov
-	/// </author>
-	/// <version>  $Id: CompoundFileReader.java 472959 2006-11-09 16:21:50Z yonik $
+	/// <version>  $Id: CompoundFileReader.java 564236 2007-08-09 15:21:19Z gsingers $
 	/// </version>
 	public class CompoundFileReader : Directory
 	{
 		
+		private int readBufferSize;
+		
 		private sealed class FileEntry
 		{
 			internal long offset;
@@ -53,16 +55,21 @@
 		private System.Collections.Hashtable entries = new System.Collections.Hashtable();
 		
 		
-		public CompoundFileReader(Directory dir, System.String name)
+		public CompoundFileReader(Directory dir, System.String name):this(dir, name, BufferedIndexInput.BUFFER_SIZE)
+		{
+		}
+		
+		public CompoundFileReader(Directory dir, System.String name, int readBufferSize)
 		{
 			directory = dir;
 			fileName = name;
+			this.readBufferSize = readBufferSize;
 			
 			bool success = false;
 			
 			try
 			{
-				stream = dir.OpenInput(name);
+                stream = dir.OpenInput(name, readBufferSize);
 				
 				// read the directory and init files
 				int count = stream.ReadVInt();
@@ -133,14 +140,23 @@
 		{
 			lock (this)
 			{
+				// Default to readBufferSize passed in when we were opened
+				return OpenInput(id, readBufferSize);
+			}
+		}
+
+        public override IndexInput OpenInput(System.String id, int readBufferSize)
+		{
+			lock (this)
+			{
 				if (stream == null)
 					throw new System.IO.IOException("Stream closed");
 				
 				FileEntry entry = (FileEntry) entries[id];
 				if (entry == null)
 					throw new System.IO.IOException("No sub-file with id " + id + " found");
-				
-				return new CSIndexInput(stream, entry.offset, entry.length);
+
+                return new CSIndexInput(stream, entry.offset, entry.length, readBufferSize);
 			}
 		}
 		
@@ -220,7 +236,11 @@
 			internal long fileOffset;
 			internal long length;
 			
-			internal CSIndexInput(IndexInput base_Renamed, long fileOffset, long length)
+			internal CSIndexInput(IndexInput base_Renamed, long fileOffset, long length) : this(base_Renamed, fileOffset, length, BufferedIndexInput.BUFFER_SIZE)
+			{
+			}
+			
+			internal CSIndexInput(IndexInput base_Renamed, long fileOffset, long length, int readBufferSize) : base(readBufferSize)
 			{
 				this.base_Renamed = base_Renamed;
 				this.fileOffset = fileOffset;
@@ -236,7 +256,7 @@
 			/// </param>
 			/// <param name="len">the number of bytes to read
 			/// </param>
-			public override void  ReadInternal(byte[] b, int offset, int len)
+			protected internal override void ReadInternal(byte[] b, int offset, int len)
 			{
 				lock (base_Renamed)
 				{
@@ -244,16 +264,16 @@
 					if (start + len > length)
 						throw new System.IO.IOException("read past EOF");
 					base_Renamed.Seek(fileOffset + start);
-					base_Renamed.ReadBytes(b, offset, len);
+                    base_Renamed.ReadBytes(b, offset, len, false);
 				}
 			}
 			
 			/// <summary>Expert: implements seek.  Sets current position in this file, where
 			/// the next {@link #ReadInternal(byte[],int,int)} will occur.
 			/// </summary>
-			/// <seealso cref="#ReadInternal(byte[],int,int)">
+			/// <seealso cref="ReadInternal(byte[],int,int)">
 			/// </seealso>
-			public override void  SeekInternal(long pos)
+			protected internal override void  SeekInternal(long pos)
 			{
 			}
 			

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/CompoundFileWriter.cs?rev=671404&r1=671403&r2=671404&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs Tue Jun 24 19:52:22 2008
@@ -16,9 +16,10 @@
  */
 
 using System;
+
 using Directory = Lucene.Net.Store.Directory;
-using IndexOutput = Lucene.Net.Store.IndexOutput;
 using IndexInput = Lucene.Net.Store.IndexInput;
+using IndexOutput = Lucene.Net.Store.IndexOutput;
 
 namespace Lucene.Net.Index
 {
@@ -43,10 +44,9 @@
 	/// contains a long pointer to the start of this file's data section, and a String
 	/// with that file's name.
 	/// 
+	/// 
 	/// </summary>
-	/// <author>  Dmitry Serebrennikov
-	/// </author>
-	/// <version>  $Id: CompoundFileWriter.java 472959 2006-11-09 16:21:50Z yonik $
+	/// <version>  $Id: CompoundFileWriter.java 606441 2007-12-22 10:06:28Z mikemccand $
 	/// </version>
 	public sealed class CompoundFileWriter
 	{
@@ -69,19 +69,23 @@
 		private System.Collections.Hashtable ids;
 		private System.Collections.ArrayList entries;
 		private bool merged = false;
-		
+		private SegmentMerger.CheckAbort checkAbort;
 		
 		/// <summary>Create the compound stream in the specified file. The file name is the
 		/// entire name (no extensions are added).
 		/// </summary>
 		/// <throws>  NullPointerException if <code>dir</code> or <code>name</code> is null </throws>
-		public CompoundFileWriter(Directory dir, System.String name)
+		public CompoundFileWriter(Directory dir, System.String name) : this(dir, name, null)
+		{
+		}
+		
+		internal CompoundFileWriter(Directory dir, System.String name, SegmentMerger.CheckAbort checkAbort)
 		{
 			if (dir == null)
 				throw new System.NullReferenceException("directory cannot be null");
 			if (name == null)
 				throw new System.NullReferenceException("name cannot be null");
-			
+			this.checkAbort = checkAbort;
 			directory = dir;
 			fileName = name;
 			ids = new System.Collections.Hashtable();
@@ -172,7 +176,7 @@
 				
 				// Open the files and copy their data into the stream.
 				// Remember the locations of each file's data section.
-				byte[] buffer = new byte[1024];
+				byte[] buffer = new byte[16384];
 				it = entries.GetEnumerator();
 				while (it.MoveNext())
 				{
@@ -233,6 +237,10 @@
 					is_Renamed.ReadBytes(buffer, 0, len);
 					os.WriteBytes(buffer, len);
 					remainder -= len;
+					if (checkAbort != null)
+					// Roughly every 2 MB we will check if
+					// it's time to abort
+						checkAbort.Work(80);
 				}
 				
 				// Verify that remainder is 0

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using Directory = Lucene.Net.Store.Directory;
+
+namespace Lucene.Net.Index
+{
+	
+	/// <summary>A {@link MergeScheduler} that runs each merge using a
+	/// separate thread, up until a maximum number of threads
+	/// ({@link #setMaxThreadCount}) at which points merges are
+	/// run in the foreground, serially.  This is a simple way
+	/// to use concurrency in the indexing process without
+	/// having to create and manage application level
+	/// threads. 
+	/// </summary>
+	
+	public class ConcurrentMergeScheduler : MergeScheduler
+	{
+		
+		private int mergeThreadPriority = - 1;
+		
+		private System.Collections.IList mergeThreads = new System.Collections.ArrayList();
+		private int maxThreadCount = 3;
+		
+		private System.Collections.IList exceptions = new System.Collections.ArrayList();
+		private Directory dir;
+		
+		private bool closed;
+		private IndexWriter writer;
+		
+		public ConcurrentMergeScheduler()
+		{
+			if (allInstances != null)
+			{
+				// Only for testing
+				AddMyself();
+			}
+		}
+		
+		/// <summary>Sets the max # simultaneous threads that may be
+		/// running.  If a merge is necessary yet we already have
+		/// this many threads running, the merge is returned back
+		/// to IndexWriter so that it runs in the "foreground". 
+		/// </summary>
+		public virtual void  SetMaxThreadCount(int count)
+		{
+			if (count < 1)
+				throw new System.ArgumentException("count should be at least 1");
+			maxThreadCount = count;
+		}
+		
+		/// <summary>Get the max # simultaneous threads that may be</summary>
+		/// <seealso cref="setMaxThreadCount.">
+		/// </seealso>
+		public virtual int GetMaxThreadCount()
+		{
+			return maxThreadCount;
+		}
+		
+		/// <summary>Return the priority that merge threads run at.  By
+		/// default the priority is 1 plus the priority of (ie,
+		/// slightly higher priority than) the first thread that
+		/// calls merge. 
+		/// </summary>
+		public virtual int GetMergeThreadPriority()
+		{
+			lock (this)
+			{
+				InitMergeThreadPriority();
+				return mergeThreadPriority;
+			}
+		}
+		
+		/// <summary>Return the priority that merge threads run at. </summary>
+		public virtual void  SetMergeThreadPriority(int pri)
+		{
+			lock (this)
+			{
+				if (pri > (int) System.Threading.ThreadPriority.Highest || pri < (int) System.Threading.ThreadPriority.Lowest)
+					throw new System.ArgumentException("priority must be in range " + (int) System.Threading.ThreadPriority.Lowest + " .. " + (int) System.Threading.ThreadPriority.Highest + " inclusive");
+				mergeThreadPriority = pri;
+				
+				int numThreads = MergeThreadCount();
+				for (int i = 0; i < numThreads; i++)
+				{
+					MergeThread merge = (MergeThread) mergeThreads[i];
+					merge.SetThreadPriority(pri);
+				}
+			}
+		}
+		
+		private void  Message(System.String message)
+		{
+			if (writer != null)
+				writer.Message("CMS: " + message);
+		}
+		
+		private void  InitMergeThreadPriority()
+		{
+			lock (this)
+			{
+				if (mergeThreadPriority == - 1)
+				{
+					// Default to slightly higher priority than our
+					// calling thread
+					mergeThreadPriority = 1 + (System.Int32) SupportClass.ThreadClass.Current().Priority;
+					if (mergeThreadPriority > (int) System.Threading.ThreadPriority.Highest)
+						mergeThreadPriority = (int) System.Threading.ThreadPriority.Highest;
+				}
+			}
+		}
+		
+		public override void  Close()
+		{
+			closed = true;
+		}
+		
+		public virtual void  Sync()
+		{
+			lock (this)
+			{
+				while (MergeThreadCount() > 0)
+				{
+					Message("now wait for threads; currently " + mergeThreads.Count + " still running");
+					int count = mergeThreads.Count;
+					for (int i = 0; i < count; i++)
+						Message("    " + i + ": " + ((MergeThread) mergeThreads[i]));
+					
+					try
+					{
+						System.Threading.Monitor.Wait(this);
+					}
+					catch (System.Threading.ThreadInterruptedException)
+					{
+					}
+				}
+			}
+		}
+		
+		private int MergeThreadCount()
+		{
+			lock (this)
+			{
+				int count = 0;
+				int numThreads = mergeThreads.Count;
+				for (int i = 0; i < numThreads; i++)
+					if (((MergeThread) mergeThreads[i]).IsAlive)
+						count++;
+				return count;
+			}
+		}
+		
+		public override void  Merge(IndexWriter writer)
+		{
+			
+			this.writer = writer;
+			
+			InitMergeThreadPriority();
+			
+			dir = writer.GetDirectory();
+			
+			// First, quickly run through the newly proposed merges
+			// and add any orthogonal merges (ie a merge not
+			// involving segments already pending to be merged) to
+			// the queue.  If we are way behind on merging, many of
+			// these newly proposed merges will likely already be
+			// registered.
+			
+			Message("now merge");
+			Message("  index: " + writer.SegString());
+			
+			// Iterate, pulling from the IndexWriter's queue of
+			// pending merges, until its empty:
+			while (true)
+			{
+				
+				// TODO: we could be careful about which merges to do in
+				// the BG (eg maybe the "biggest" ones) vs FG, which
+				// merges to do first (the easiest ones?), etc.
+				
+				MergePolicy.OneMerge merge = writer.GetNextMerge();
+				if (merge == null)
+				{
+					Message("  no more merges pending; now return");
+					return ;
+				}
+				
+				// We do this w/ the primary thread to keep
+				// deterministic assignment of segment names
+				writer.MergeInit(merge);
+				
+				Message("  consider merge " + merge.SegString(dir));
+				
+				if (merge.isExternal)
+				{
+					Message("    merge involves segments from an external directory; now run in foreground");
+				}
+				else
+				{
+					lock (this)
+					{
+						if (MergeThreadCount() < maxThreadCount)
+						{
+							// OK to spawn a new merge thread to handle this
+							// merge:
+							MergeThread merger = new MergeThread(this, writer, merge);
+							mergeThreads.Add(merger);
+							Message("    launch new thread [" + merger.Name + "]");
+							merger.SetThreadPriority(mergeThreadPriority);
+							merger.IsBackground = true;
+							merger.Start();
+							continue;
+						}
+						else
+							Message("    too many merge threads running; run merge in foreground");
+					}
+				}
+				
+				// Too many merge threads already running, so we do
+				// this in the foreground of the calling thread
+				writer.Merge(merge);
+			}
+		}
+		
+		private class MergeThread:SupportClass.ThreadClass
+		{
+			private void  InitBlock(ConcurrentMergeScheduler enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private ConcurrentMergeScheduler enclosingInstance;
+			public ConcurrentMergeScheduler Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			
+			internal IndexWriter writer;
+			internal MergePolicy.OneMerge startMerge;
+			internal MergePolicy.OneMerge runningMerge;
+			
+			public MergeThread(ConcurrentMergeScheduler enclosingInstance, IndexWriter writer, MergePolicy.OneMerge startMerge)
+			{
+				InitBlock(enclosingInstance);
+				this.writer = writer;
+				this.startMerge = startMerge;
+			}
+			
+			public virtual void  SetRunningMerge(MergePolicy.OneMerge merge)
+			{
+				lock (this)
+				{
+					runningMerge = merge;
+				}
+			}
+			
+			public virtual MergePolicy.OneMerge GetRunningMerge()
+			{
+				lock (this)
+				{
+					return runningMerge;
+				}
+			}
+			
+			public virtual void  SetThreadPriority(int pri)
+			{
+				try
+				{
+					Priority = (System.Threading.ThreadPriority) pri;
+				}
+				catch (System.NullReferenceException)
+				{
+					// Strangely, Sun's JDK 1.5 on Linux sometimes
+					// throws NPE out of here...
+				}
+				catch (System.Security.SecurityException)
+				{
+					// Ignore this because we will still run fine with
+					// normal thread priority
+				}
+			}
+			
+			override public void  Run()
+			{
+				
+				// First time through the while loop we do the merge
+				// that we were started with:
+				MergePolicy.OneMerge merge = this.startMerge;
+				
+				try
+				{
+					
+					Enclosing_Instance.Message("  merge thread: start");
+					
+					while (true)
+					{
+						SetRunningMerge(merge);
+						writer.Merge(merge);
+						
+						// Subsequent times through the loop we do any new
+						// merge that writer says is necessary:
+						merge = writer.GetNextMerge();
+						if (merge != null)
+						{
+							writer.MergeInit(merge);
+							Enclosing_Instance.Message("  merge thread: do another merge " + merge.SegString(Enclosing_Instance.dir));
+						}
+						else
+							break;
+					}
+					
+					Enclosing_Instance.Message("  merge thread: done");
+				}
+				catch (System.Exception exc)
+				{
+					
+					if (merge != null)
+					{
+						merge.SetException(exc);
+						writer.AddMergeException(merge);
+					}
+					
+					// Ignore the exception if it was due to abort:
+					if (!(exc is MergePolicy.MergeAbortedException))
+					{
+						lock (Enclosing_Instance)
+						{
+							Enclosing_Instance.exceptions.Add(exc);
+						}
+						
+						if (!Enclosing_Instance.suppressExceptions)
+						{
+							// suppressExceptions is normally only set during
+							// testing.
+							Lucene.Net.Index.ConcurrentMergeScheduler.anyExceptions = true;
+							throw new MergePolicy.MergeException(exc);
+						}
+					}
+				}
+				finally
+				{
+					lock (Enclosing_Instance)
+					{
+						Enclosing_Instance.mergeThreads.Remove(this);
+						System.Threading.Monitor.PulseAll(Enclosing_Instance);
+					}
+				}
+			}
+			
+			public override System.String ToString()
+			{
+				MergePolicy.OneMerge merge = GetRunningMerge();
+				if (merge == null)
+					merge = startMerge;
+				return "merge thread: " + merge.SegString(Enclosing_Instance.dir);
+			}
+		}
+		
+		internal static bool anyExceptions = false;
+		
+		/// <summary>Used for testing </summary>
+		public static bool AnyUnhandledExceptions()
+		{
+			lock (allInstances.SyncRoot)
+			{
+				int count = allInstances.Count;
+				// Make sure all outstanding threads are done so we see
+				// any exceptions they may produce:
+				for (int i = 0; i < count; i++)
+					((ConcurrentMergeScheduler) allInstances[i]).Sync();
+				return anyExceptions;
+			}
+		}
+		
+		/// <summary>Used for testing </summary>
+		private void  AddMyself()
+		{
+			lock (allInstances.SyncRoot)
+			{
+				int size = 0;
+				int upto = 0;
+				for (int i = 0; i < size; i++)
+				{
+					ConcurrentMergeScheduler other = (ConcurrentMergeScheduler) allInstances[i];
+					if (!(other.closed && 0 == other.MergeThreadCount()))
+					// Keep this one for now: it still has threads or
+					// may spawn new threads
+						allInstances[upto++] = other;
+				}
+				((System.Collections.IList) ((System.Collections.ArrayList) allInstances).GetRange(upto, allInstances.Count - upto)).Clear();
+				allInstances.Add(this);
+			}
+		}
+		
+		private bool suppressExceptions;
+		
+		/// <summary>Used for testing </summary>
+		internal virtual void  SetSuppressExceptions()
+		{
+			suppressExceptions = true;
+		}
+		
+		/// <summary>Used for testing </summary>
+		internal virtual void  ClearSuppressExceptions()
+		{
+			suppressExceptions = false;
+		}
+		
+		/// <summary>Used for testing </summary>
+		private static System.Collections.IList allInstances;
+		public static void  SetTestMode()
+		{
+			allInstances = new System.Collections.ArrayList();
+		}
+	}
+}
\ No newline at end of file

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CorruptIndexException.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/CorruptIndexException.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CorruptIndexException.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CorruptIndexException.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+namespace Lucene.Net.Index
+{
+	
+	/// <summary> This exception is thrown when Lucene detects
+	/// an inconsistency in the index.
+	/// </summary>
+	[Serializable]
+	public class CorruptIndexException : System.IO.IOException
+	{
+		public CorruptIndexException(System.String message) : base(message)
+		{
+		}
+	}
+}
\ No newline at end of file

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DefaultSkipListReader.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListReader.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListReader.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using IndexInput = Lucene.Net.Store.IndexInput;
+
+namespace Lucene.Net.Index
+{
+	
+	/// <summary> Implements the skip list reader for the default posting list format
+	/// that stores positions and payloads.
+	/// 
+	/// </summary>
+	class DefaultSkipListReader : MultiLevelSkipListReader
+	{
+		private bool currentFieldStoresPayloads;
+		private long[] freqPointer;
+		private long[] proxPointer;
+		private int[] payloadLength;
+		
+		private long lastFreqPointer;
+		private long lastProxPointer;
+		private int lastPayloadLength;
+		
+		
+		internal DefaultSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) : base(skipStream, maxSkipLevels, skipInterval)
+		{
+			freqPointer = new long[maxSkipLevels];
+			proxPointer = new long[maxSkipLevels];
+			payloadLength = new int[maxSkipLevels];
+		}
+		
+		internal virtual void  Init(long skipPointer, long freqBasePointer, long proxBasePointer, int df, bool storesPayloads)
+		{
+			base.Init(skipPointer, df);
+			this.currentFieldStoresPayloads = storesPayloads;
+			lastFreqPointer = freqBasePointer;
+			lastProxPointer = proxBasePointer;
+
+            for (int i = 0; i < freqPointer.Length; i++) freqPointer[i] = freqBasePointer;
+            for (int i = 0; i < proxPointer.Length; i++) proxPointer[i] = proxBasePointer;
+            for (int i = 0; i < payloadLength.Length; i++) payloadLength[i] = 0;
+		}
+		
+		/// <summary>Returns the freq pointer of the doc to which the last call of 
+		/// {@link MultiLevelSkipListReader#SkipTo(int)} has skipped.  
+		/// </summary>
+		internal virtual long GetFreqPointer()
+		{
+			return lastFreqPointer;
+		}
+		
+		/// <summary>Returns the prox pointer of the doc to which the last call of 
+		/// {@link MultiLevelSkipListReader#SkipTo(int)} has skipped.  
+		/// </summary>
+		internal virtual long GetProxPointer()
+		{
+			return lastProxPointer;
+		}
+		
+		/// <summary>Returns the payload length of the payload stored just before 
+		/// the doc to which the last call of {@link MultiLevelSkipListReader#SkipTo(int)} 
+		/// has skipped.  
+		/// </summary>
+		internal virtual int GetPayloadLength()
+		{
+			return lastPayloadLength;
+		}
+		
+		protected internal override void  SeekChild(int level)
+		{
+			base.SeekChild(level);
+			freqPointer[level] = lastFreqPointer;
+			proxPointer[level] = lastProxPointer;
+			payloadLength[level] = lastPayloadLength;
+		}
+		
+		protected internal override void  SetLastSkipData(int level)
+		{
+			base.SetLastSkipData(level);
+			lastFreqPointer = freqPointer[level];
+			lastProxPointer = proxPointer[level];
+			lastPayloadLength = payloadLength[level];
+		}
+		
+		
+		protected internal override int ReadSkipData(int level, IndexInput skipStream)
+		{
+			int delta;
+			if (currentFieldStoresPayloads)
+			{
+				// the current field stores payloads.
+				// if the doc delta is odd then we have
+				// to read the current payload length
+				// because it differs from the length of the
+				// previous payload
+				delta = skipStream.ReadVInt();
+				if ((delta & 1) != 0)
+				{
+					payloadLength[level] = skipStream.ReadVInt();
+				}
+				delta = SupportClass.Number.URShift(delta, 1);
+			}
+			else
+			{
+				delta = skipStream.ReadVInt();
+			}
+			freqPointer[level] += skipStream.ReadVInt();
+			proxPointer[level] += skipStream.ReadVInt();
+			
+			return delta;
+		}
+	}
+}
\ No newline at end of file

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DefaultSkipListWriter.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using IndexOutput = Lucene.Net.Store.IndexOutput;
+
+namespace Lucene.Net.Index
+{
+	
+	
+	/// <summary> Implements the skip list writer for the default posting list format
+	/// that stores positions and payloads.
+	/// 
+	/// </summary>
+	class DefaultSkipListWriter : MultiLevelSkipListWriter
+	{
+		private int[] lastSkipDoc;
+		private int[] lastSkipPayloadLength;
+		private long[] lastSkipFreqPointer;
+		private long[] lastSkipProxPointer;
+		
+		private IndexOutput freqOutput;
+		private IndexOutput proxOutput;
+		
+		private int curDoc;
+		private bool curStorePayloads;
+		private int curPayloadLength;
+		private long curFreqPointer;
+		private long curProxPointer;
+		
+		internal DefaultSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput, IndexOutput proxOutput) : base(skipInterval, numberOfSkipLevels, docCount)
+		{
+			this.freqOutput = freqOutput;
+			this.proxOutput = proxOutput;
+			
+			lastSkipDoc = new int[numberOfSkipLevels];
+			lastSkipPayloadLength = new int[numberOfSkipLevels];
+			lastSkipFreqPointer = new long[numberOfSkipLevels];
+			lastSkipProxPointer = new long[numberOfSkipLevels];
+		}
+		
+		/// <summary> Sets the values for the current skip data. </summary>
+		internal virtual void  SetSkipData(int doc, bool storePayloads, int payloadLength)
+		{
+			this.curDoc = doc;
+			this.curStorePayloads = storePayloads;
+			this.curPayloadLength = payloadLength;
+			this.curFreqPointer = freqOutput.GetFilePointer();
+			this.curProxPointer = proxOutput.GetFilePointer();
+		}
+		
+		protected internal override void  ResetSkip()
+		{
+			base.ResetSkip();
+			for (int i = 0; i < lastSkipDoc.Length; i++) lastSkipDoc[i] = 0;
+			for (int i = 0; i < lastSkipPayloadLength.Length; i++) lastSkipPayloadLength[i] = -1; // we don't have to write the first length in the skip list
+			for (int i = 0; i < lastSkipFreqPointer.Length; i++) lastSkipFreqPointer[i] = freqOutput.GetFilePointer();
+			for (int i = 0; i < lastSkipProxPointer.Length; i++) lastSkipProxPointer[i] = proxOutput.GetFilePointer();
+		}
+		
+		protected internal override void  WriteSkipData(int level, IndexOutput skipBuffer)
+		{
+			// To efficiently store payloads in the posting lists we do not store the length of
+			// every payload. Instead we omit the length for a payload if the previous payload had
+			// the same length.
+			// However, in order to support skipping the payload length at every skip point must be known.
+			// So we use the same length encoding that we use for the posting lists for the skip data as well:
+			// Case 1: current field does not store payloads
+			//           SkipDatum                 --> DocSkip, FreqSkip, ProxSkip
+			//           DocSkip,FreqSkip,ProxSkip --> VInt
+			//           DocSkip records the document number before every SkipInterval th  document in TermFreqs. 
+			//           Document numbers are represented as differences from the previous value in the sequence.
+			// Case 2: current field stores payloads
+			//           SkipDatum                 --> DocSkip, PayloadLength?, FreqSkip,ProxSkip
+			//           DocSkip,FreqSkip,ProxSkip --> VInt
+			//           PayloadLength             --> VInt    
+			//         In this case DocSkip/2 is the difference between
+			//         the current and the previous value. If DocSkip
+			//         is odd, then a PayloadLength encoded as VInt follows,
+			//         if DocSkip is even, then it is assumed that the
+			//         current payload length equals the length at the previous
+			//         skip point
+			if (curStorePayloads)
+			{
+				int delta = curDoc - lastSkipDoc[level];
+				if (curPayloadLength == lastSkipPayloadLength[level])
+				{
+					// the current payload length equals the length at the previous skip point,
+					// so we don't store the length again
+					skipBuffer.WriteVInt(delta * 2);
+				}
+				else
+				{
+					// the payload length is different from the previous one. We shift the DocSkip, 
+					// set the lowest bit and store the current payload length as VInt.
+					skipBuffer.WriteVInt(delta * 2 + 1);
+					skipBuffer.WriteVInt(curPayloadLength);
+					lastSkipPayloadLength[level] = curPayloadLength;
+				}
+			}
+			else
+			{
+				// current field does not store payloads
+				skipBuffer.WriteVInt(curDoc - lastSkipDoc[level]);
+			}
+			skipBuffer.WriteVInt((int) (curFreqPointer - lastSkipFreqPointer[level]));
+			skipBuffer.WriteVInt((int) (curProxPointer - lastSkipProxPointer[level]));
+			
+			lastSkipDoc[level] = curDoc;
+			//System.out.println("write doc at level " + level + ": " + curDoc);
+			
+			lastSkipFreqPointer[level] = curFreqPointer;
+			lastSkipProxPointer[level] = curProxPointer;
+		}
+	}
+}
\ No newline at end of file

Added: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DirectoryIndexReader.cs?rev=671404&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs Tue Jun 24 19:52:22 2008
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using Directory = Lucene.Net.Store.Directory;
+using Lock = Lucene.Net.Store.Lock;
+using LockObtainFailedException = Lucene.Net.Store.LockObtainFailedException;
+
+namespace Lucene.Net.Index
+{
+	
+	/// <summary> IndexReader implementation that has access to a Directory. 
+	/// Instances that have a SegmentInfos object (i. e. segmentInfos != null)
+	/// "own" the directory, which means that they try to acquire a write lock
+	/// whenever index modifications are performed.
+	/// </summary>
+	abstract public class DirectoryIndexReader : IndexReader
+	{
+		private class AnonymousClassFindSegmentsFile : SegmentInfos.FindSegmentsFile
+		{
+			private void  InitBlock(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy)
+			{
+				this.closeDirectory = closeDirectory;
+				this.deletionPolicy = deletionPolicy;
+			}
+			private bool closeDirectory;
+			private Lucene.Net.Index.IndexDeletionPolicy deletionPolicy;
+			internal AnonymousClassFindSegmentsFile(bool closeDirectory, Lucene.Net.Index.IndexDeletionPolicy deletionPolicy, Lucene.Net.Store.Directory Param1) : base(Param1)
+			{
+				InitBlock(closeDirectory, deletionPolicy);
+			}
+			
+			protected internal override System.Object DoBody(System.String segmentFileName)
+			{
+				
+				SegmentInfos infos = new SegmentInfos();
+				infos.Read(directory, segmentFileName);
+				
+				DirectoryIndexReader reader;
+				
+				if (infos.Count == 1)
+				{
+					// index is optimized
+					reader = SegmentReader.Get(infos, infos.Info(0), closeDirectory);
+				}
+				else
+				{
+					reader = new MultiSegmentReader(directory, infos, closeDirectory);
+				}
+				reader.SetDeletionPolicy(deletionPolicy);
+				return reader;
+			}
+		}
+		
+		private class AnonymousClassFindSegmentsFile1 : SegmentInfos.FindSegmentsFile
+		{
+			private void  InitBlock(DirectoryIndexReader enclosingInstance)
+			{
+				this.enclosingInstance = enclosingInstance;
+			}
+			private DirectoryIndexReader enclosingInstance;
+			public DirectoryIndexReader Enclosing_Instance
+			{
+				get
+				{
+					return enclosingInstance;
+				}
+				
+			}
+			internal AnonymousClassFindSegmentsFile1(DirectoryIndexReader enclosingInstance, Lucene.Net.Store.Directory Param1) : base(Param1)
+			{
+				InitBlock(enclosingInstance);
+			}
+			
+			protected internal override System.Object DoBody(System.String segmentFileName)
+			{
+				SegmentInfos infos = new SegmentInfos();
+				infos.Read(directory, segmentFileName);
+				
+				DirectoryIndexReader newReader = Enclosing_Instance.DoReopen(infos);
+				
+				if (Enclosing_Instance != newReader)
+				{
+					newReader.Init(directory, infos, Enclosing_Instance.closeDirectory);
+					newReader.deletionPolicy = Enclosing_Instance.deletionPolicy;
+				}
+				
+				return newReader;
+			}
+		}
+		protected internal Directory directory;
+		protected internal bool closeDirectory;
+		private IndexDeletionPolicy deletionPolicy;
+		
+		private SegmentInfos segmentInfos;
+		private Lock writeLock;
+		private bool stale;
+		
+		/// <summary>Used by commit() to record pre-commit state in case
+		/// rollback is necessary 
+		/// </summary>
+		private bool rollbackHasChanges;
+		private SegmentInfos rollbackSegmentInfos;
+		
+		
+		internal virtual void  Init(Directory directory, SegmentInfos segmentInfos, bool closeDirectory)
+		{
+			this.directory = directory;
+			this.segmentInfos = segmentInfos;
+			this.closeDirectory = closeDirectory;
+		}
+		
+		protected internal DirectoryIndexReader()
+		{
+		}
+		
+		internal DirectoryIndexReader(Directory directory, SegmentInfos segmentInfos, bool closeDirectory) : base()
+		{
+			Init(directory, segmentInfos, closeDirectory);
+		}
+		
+		internal static DirectoryIndexReader Open(Directory directory, bool closeDirectory, IndexDeletionPolicy deletionPolicy)
+		{
+			
+			return (DirectoryIndexReader) new AnonymousClassFindSegmentsFile(closeDirectory, deletionPolicy, directory).Run();
+		}
+		
+		
+		public override IndexReader Reopen()
+		{
+			lock (this)
+			{
+				EnsureOpen();
+				
+				if (this.hasChanges || this.IsCurrent())
+				{
+					// the index hasn't changed - nothing to do here
+					return this;
+				}
+				
+				return (DirectoryIndexReader) new AnonymousClassFindSegmentsFile1(this, directory).Run();
+			}
+		}
+		
+		/// <summary> Re-opens the index using the passed-in SegmentInfos </summary>
+		protected internal abstract DirectoryIndexReader DoReopen(SegmentInfos infos);
+		
+		public virtual void  SetDeletionPolicy(IndexDeletionPolicy deletionPolicy)
+		{
+			this.deletionPolicy = deletionPolicy;
+		}
+		
+		/// <summary>Returns the directory this index resides in.</summary>
+		public override Directory Directory()
+		{
+			EnsureOpen();
+			return directory;
+		}
+		
+		/// <summary> Version number when this IndexReader was opened.</summary>
+		public override long GetVersion()
+		{
+			EnsureOpen();
+			return segmentInfos.GetVersion();
+		}
+		
+		/// <summary> Check whether this IndexReader is still using the
+		/// current (i.e., most recently committed) version of the
+		/// index.  If a writer has committed any changes to the
+		/// index since this reader was opened, this will return
+		/// <code>false</code>, in which case you must open a new
+		/// IndexReader in order to see the changes.  See the
+		/// description of the <a href="IndexWriter.html#autoCommit"><code>autoCommit</code></a>
+		/// flag which controls when the {@link IndexWriter}
+		/// actually commits changes to the index.
+		/// 
+		/// </summary>
+		/// <throws>  CorruptIndexException if the index is corrupt </throws>
+		/// <throws>  IOException if there is a low-level IO error </throws>
+		public override bool IsCurrent()
+		{
+			EnsureOpen();
+			return SegmentInfos.ReadCurrentVersion(directory) == segmentInfos.GetVersion();
+		}
+		
+		/// <summary> Checks is the index is optimized (if it has a single segment and no deletions)</summary>
+		/// <returns> <code>true</code> if the index is optimized; <code>false</code> otherwise
+		/// </returns>
+		public override bool IsOptimized()
+		{
+			EnsureOpen();
+			return segmentInfos.Count == 1 && HasDeletions() == false;
+		}
+		
+		protected internal override void  DoClose()
+		{
+			if (closeDirectory)
+				directory.Close();
+		}
+		
+		/// <summary> Commit changes resulting from delete, undeleteAll, or
+		/// setNorm operations
+		/// 
+		/// If an exception is hit, then either no changes or all
+		/// changes will have been committed to the index
+		/// (transactional semantics).
+		/// </summary>
+		/// <throws>  IOException if there is a low-level IO error </throws>
+		protected internal override void  DoCommit()
+		{
+			if (hasChanges)
+			{
+				if (segmentInfos != null)
+				{
+					
+					// Default deleter (for backwards compatibility) is
+					// KeepOnlyLastCommitDeleter:
+					IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, segmentInfos, null, null);
+					
+					// Checkpoint the state we are about to change, in
+					// case we have to roll back:
+					StartCommit();
+					
+					bool success = false;
+					try
+					{
+						CommitChanges();
+						segmentInfos.Write(directory);
+						success = true;
+					}
+					finally
+					{
+						
+						if (!success)
+						{
+							
+							// Rollback changes that were made to
+							// SegmentInfos but failed to get [fully]
+							// committed.  This way this reader instance
+							// remains consistent (matched to what's
+							// actually in the index):
+							RollbackCommit();
+							
+							// Recompute deletable files & remove them (so
+							// partially written .del files, etc, are
+							// removed):
+							deleter.Refresh();
+						}
+					}
+					
+					// Have the deleter remove any now unreferenced
+					// files due to this commit:
+					deleter.Checkpoint(segmentInfos, true);
+					
+					if (writeLock != null)
+					{
+						writeLock.Release(); // release write lock
+						writeLock = null;
+					}
+				}
+				else
+					CommitChanges();
+			}
+			hasChanges = false;
+		}
+		
+		protected internal abstract void  CommitChanges();
+		
+		/// <summary> Tries to acquire the WriteLock on this directory.
+		/// this method is only valid if this IndexReader is directory owner.
+		/// 
+		/// </summary>
+		/// <throws>  StaleReaderException if the index has changed </throws>
+		/// <summary> since this reader was opened
+		/// </summary>
+		/// <throws>  CorruptIndexException if the index is corrupt </throws>
+		/// <throws>  LockObtainFailedException if another writer </throws>
+		/// <summary>  has this index open (<code>write.lock</code> could not
+		/// be obtained)
+		/// </summary>
+		/// <throws>  IOException if there is a low-level IO error </throws>
+		protected internal override void  AcquireWriteLock()
+		{
+			if (segmentInfos != null)
+			{
+				EnsureOpen();
+				if (stale)
+					throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
+				
+				if (this.writeLock == null)
+				{
+					Lock writeLock = directory.MakeLock(IndexWriter.WRITE_LOCK_NAME);
+					if (!writeLock.Obtain(IndexWriter.WRITE_LOCK_TIMEOUT))
+					// obtain write lock
+					{
+						throw new LockObtainFailedException("Index locked for write: " + writeLock);
+					}
+					this.writeLock = writeLock;
+					
+					// we have to check whether index has changed since this reader was opened.
+					// if so, this reader is no longer valid for deletion
+					if (SegmentInfos.ReadCurrentVersion(directory) > segmentInfos.GetVersion())
+					{
+						stale = true;
+						this.writeLock.Release();
+						this.writeLock = null;
+						throw new StaleReaderException("IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
+					}
+				}
+			}
+		}
+		
+		/// <summary> Should internally checkpoint state that will change
+		/// during commit so that we can rollback if necessary.
+		/// </summary>
+		internal virtual void  StartCommit()
+		{
+			if (segmentInfos != null)
+			{
+				rollbackSegmentInfos = (SegmentInfos) segmentInfos.Clone();
+			}
+			rollbackHasChanges = hasChanges;
+		}
+		
+		/// <summary> Rolls back state to just before the commit (this is
+		/// called by commit() if there is some exception while
+		/// committing).
+		/// </summary>
+		internal virtual void  RollbackCommit()
+		{
+			if (segmentInfos != null)
+			{
+				for (int i = 0; i < segmentInfos.Count; i++)
+				{
+					// Rollback each segmentInfo.  Because the
+					// SegmentReader holds a reference to the
+					// SegmentInfo we can't [easily] just replace
+					// segmentInfos, so we reset it in place instead:
+					segmentInfos.Info(i).Reset(rollbackSegmentInfos.Info(i));
+				}
+				rollbackSegmentInfos = null;
+			}
+			
+			hasChanges = rollbackHasChanges;
+		}
+		
+		/// <summary>Release the write lock, if needed. </summary>
+		~DirectoryIndexReader()
+		{
+			try
+			{
+				if (writeLock != null)
+				{
+					writeLock.Release(); // release write lock
+					writeLock = null;
+				}
+			}
+			finally
+            {
+				// {{Aroush-2.3.1}} do we need to call Finalize() here?
+            }
+		}
+	}
+}
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Index/DocumentWriter.cs?rev=671404&r1=671403&r2=671404&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs Tue Jun 24 19:52:22 2008
@@ -14,7 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+// {{Aroush-2.3.1}} remove this file from SVN
+/*
 using System;
 using Analyzer = Lucene.Net.Analysis.Analyzer;
 using Token = Lucene.Net.Analysis.Token;
@@ -509,4 +510,4 @@
 				offsets = null;
 		}
 	}
-}
\ No newline at end of file
+}*/
\ No newline at end of file



Mime
View raw message