lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dougs...@apache.org
Subject svn commit: r798995 [28/35] - in /incubator/lucene.net/trunk/C#/src: Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Analysis/Standard/ Lucene.Net/Document/ Lucene.Net/Index/ Lucene.Net/QueryParser/ Lucene.Net/Search/ Lucene.Net/Search/Function/ Lucene.Net...
Date Wed, 29 Jul 2009 18:04:24 GMT
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterDelete.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterDelete.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterDelete.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterDelete.cs Wed Jul 29 18:04:12 2009
@@ -19,24 +19,21 @@
 
 using NUnit.Framework;
 
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
 using Document = Lucene.Net.Documents.Document;
 using Field = Lucene.Net.Documents.Field;
-using Directory = Lucene.Net.Store.Directory;
-using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
-using RAMDirectory = Lucene.Net.Store.RAMDirectory;
-using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
-using Hits = Lucene.Net.Search.Hits;
 using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+using ScoreDoc = Lucene.Net.Search.ScoreDoc;
 using TermQuery = Lucene.Net.Search.TermQuery;
+using Directory = Lucene.Net.Store.Directory;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
 using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
 
 namespace Lucene.Net.Index
 {
-	
 	[TestFixture]
 	public class TestIndexWriterDelete : LuceneTestCase
 	{
-		
 		private class AnonymousClassFailure : MockRAMDirectory.Failure
 		{
 			public AnonymousClassFailure(TestIndexWriterDelete enclosingInstance)
@@ -150,7 +147,7 @@
 			{
 				bool autoCommit = (0 == pass);
 				
-				Directory dir = new RAMDirectory();
+				Directory dir = new MockRAMDirectory();
 				IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
 				modifier.SetUseCompoundFile(true);
 				modifier.SetMaxBufferedDeleteTerms(1);
@@ -158,39 +155,24 @@
 				for (int i = 0; i < keywords.Length; i++)
 				{
 					Document doc = new Document();
-					doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.UN_TOKENIZED));
+					doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
 					doc.Add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO));
-					doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.TOKENIZED));
-					doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.TOKENIZED));
+					doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED));
+					doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED));
 					modifier.AddDocument(doc);
 				}
 				modifier.Optimize();
-				
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
+                modifier.Commit();
 				
 				Term term = new Term("city", "Amsterdam");
 				int hitCount = GetHitCount(dir, term);
 				Assert.AreEqual(1, hitCount);
-				if (!autoCommit)
-				{
-					modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
-					modifier.SetUseCompoundFile(true);
-				}
 				modifier.DeleteDocuments(term);
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
+                modifier.Commit();
 				hitCount = GetHitCount(dir, term);
 				Assert.AreEqual(0, hitCount);
 				
-				if (autoCommit)
-				{
-					modifier.Close();
-				}
+				modifier.Close();
 				dir.Close();
 			}
 		}
@@ -203,7 +185,7 @@
 			{
 				bool autoCommit = (0 == pass);
 				
-				Directory dir = new RAMDirectory();
+				Directory dir = new MockRAMDirectory();
 				IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
 				modifier.SetMaxBufferedDocs(2);
 				modifier.SetMaxBufferedDeleteTerms(2);
@@ -215,42 +197,25 @@
 				{
 					AddDoc(modifier, ++id, value_Renamed);
 				}
-				modifier.Flush();
+				modifier.Commit();
 				
 				Assert.AreEqual(0, modifier.GetNumBufferedDocuments());
 				Assert.IsTrue(0 < modifier.GetSegmentCount());
-				
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
+
+                modifier.Commit();
 				
 				IndexReader reader = IndexReader.Open(dir);
 				Assert.AreEqual(7, reader.NumDocs());
 				reader.Close();
 				
-				if (!autoCommit)
-				{
-					modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
-					modifier.SetMaxBufferedDocs(2);
-					modifier.SetMaxBufferedDeleteTerms(2);
-				}
-				
-				modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
 				modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
-				
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
+
+                modifier.Commit();
 				
 				reader = IndexReader.Open(dir);
 				Assert.AreEqual(0, reader.NumDocs());
 				reader.Close();
-				if (autoCommit)
-				{
-					modifier.Close();
-				}
+				modifier.Close();
 				dir.Close();
 			}
 		}
@@ -261,43 +226,47 @@
 		{
 			for (int pass = 0; pass < 2; pass++)
 			{
-				bool autoCommit = (0 == pass);
-				Directory dir = new RAMDirectory();
-				IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
-				modifier.SetMaxBufferedDocs(4);
-				modifier.SetMaxBufferedDeleteTerms(4);
-				
-				int id = 0;
-				int value_Renamed = 100;
-				
-				AddDoc(modifier, ++id, value_Renamed);
-				modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
-				AddDoc(modifier, ++id, value_Renamed);
-				modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
-				
-				Assert.AreEqual(2, modifier.GetNumBufferedDeleteTerms());
-				Assert.AreEqual(1, modifier.GetBufferedDeleteTermsSize());
-				
-				AddDoc(modifier, ++id, value_Renamed);
-				Assert.AreEqual(0, modifier.GetSegmentCount());
-				modifier.Flush();
-				
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
-				
-				IndexReader reader = IndexReader.Open(dir);
-				Assert.AreEqual(1, reader.NumDocs());
-				
-				int hitCount = GetHitCount(dir, new Term("id", System.Convert.ToString(id)));
-				Assert.AreEqual(1, hitCount);
-				reader.Close();
-				if (autoCommit)
-				{
-					modifier.Close();
-				}
-				dir.Close();
+                for (int t = 0; t < 2; t++)
+                {
+                    bool autoCommit = (0 == pass);
+                    Directory dir = new MockRAMDirectory();
+                    IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
+                    modifier.SetMaxBufferedDocs(4);
+                    modifier.SetMaxBufferedDeleteTerms(4);
+
+                    int id = 0;
+                    int value_Renamed = 100;
+
+                    AddDoc(modifier, ++id, value_Renamed);
+                    if (0 == t)
+                        modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
+                    else
+                        modifier.DeleteDocuments(new TermQuery(new Term("value", System.Convert.ToString(value_Renamed))));
+                    AddDoc(modifier, ++id, value_Renamed);
+                    if (0 == t)
+                    {
+                        modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
+                        Assert.AreEqual(2, modifier.GetNumBufferedDeleteTerms());
+                        Assert.AreEqual(1, modifier.GetBufferedDeleteTermsSize());
+                    }
+                    else
+                        modifier.DeleteDocuments(new TermQuery(new Term("value", System.Convert.ToString(value_Renamed))));
+
+                    AddDoc(modifier, ++id, value_Renamed);
+                    Assert.AreEqual(0, modifier.GetSegmentCount());
+                    modifier.Flush();
+
+                    modifier.Commit();
+
+                    IndexReader reader = IndexReader.Open(dir);
+                    Assert.AreEqual(1, reader.NumDocs());
+
+                    int hitCount = GetHitCount(dir, new Term("id", System.Convert.ToString(id)));
+                    Assert.AreEqual(1, hitCount);
+                    reader.Close();
+                    modifier.Close();
+                    dir.Close();
+                }
 			}
 		}
 		
@@ -309,7 +278,7 @@
 			{
 				bool autoCommit = (0 == pass);
 				
-				Directory dir = new RAMDirectory();
+				Directory dir = new MockRAMDirectory();
 				IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
 				modifier.SetMaxBufferedDocs(100);
 				modifier.SetMaxBufferedDeleteTerms(100);
@@ -327,26 +296,19 @@
 				{
 					AddDoc(modifier, ++id, value_Renamed);
 				}
-				modifier.Flush();
+				modifier.Commit();
 				
 				for (int i = 0; i < 5; i++)
 				{
 					AddDoc(modifier, ++id, value_Renamed);
 				}
 				modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
-				
-				modifier.Flush();
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
+
+                modifier.Commit();
 				
 				IndexReader reader = IndexReader.Open(dir);
 				Assert.AreEqual(5, reader.NumDocs());
-				if (autoCommit)
-				{
-					modifier.Close();
-				}
+				modifier.Close();
 			}
 		}
 		
@@ -357,7 +319,7 @@
 			for (int pass = 0; pass < 2; pass++)
 			{
 				bool autoCommit = (0 == pass);
-				Directory dir = new RAMDirectory();
+				Directory dir = new MockRAMDirectory();
 				IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
 				modifier.SetMaxBufferedDocs(2);
 				modifier.SetMaxBufferedDeleteTerms(2);
@@ -369,31 +331,17 @@
 				{
 					AddDoc(modifier, ++id, value_Renamed);
 				}
-				modifier.Flush();
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
+				modifier.Commit();
 				
 				IndexReader reader = IndexReader.Open(dir);
 				Assert.AreEqual(7, reader.NumDocs());
 				reader.Close();
 				
-				if (!autoCommit)
-				{
-					modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
-					modifier.SetMaxBufferedDocs(2);
-					modifier.SetMaxBufferedDeleteTerms(2);
-				}
-				
 				id = 0;
 				modifier.DeleteDocuments(new Term("id", System.Convert.ToString(++id)));
 				modifier.DeleteDocuments(new Term("id", System.Convert.ToString(++id)));
-				
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
+
+                modifier.Commit();
 				
 				reader = IndexReader.Open(dir);
 				Assert.AreEqual(5, reader.NumDocs());
@@ -404,25 +352,13 @@
 				{
 					terms[i] = new Term("id", System.Convert.ToString(++id));
 				}
-				if (!autoCommit)
-				{
-					modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
-					modifier.SetMaxBufferedDocs(2);
-					modifier.SetMaxBufferedDeleteTerms(2);
-				}
 				modifier.DeleteDocuments(terms);
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
-				reader = IndexReader.Open(dir);
+                modifier.Commit();
+                reader = IndexReader.Open(dir);
 				Assert.AreEqual(2, reader.NumDocs());
 				reader.Close();
 				
-				if (autoCommit)
-				{
-					modifier.Close();
-				}
+				modifier.Close();
 				dir.Close();
 			}
 		}
@@ -430,16 +366,16 @@
 		private void  AddDoc(IndexWriter modifier, int id, int value_Renamed)
 		{
 			Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
-			doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.TOKENIZED));
-			doc.Add(new Field("id", System.Convert.ToString(id), Field.Store.YES, Field.Index.UN_TOKENIZED));
-			doc.Add(new Field("value", System.Convert.ToString(value_Renamed), Field.Store.NO, Field.Index.UN_TOKENIZED));
+			doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
+			doc.Add(new Field("id", System.Convert.ToString(id), Field.Store.YES, Field.Index.NOT_ANALYZED));
+			doc.Add(new Field("value", System.Convert.ToString(value_Renamed), Field.Store.NO, Field.Index.NOT_ANALYZED));
 			modifier.AddDocument(doc);
 		}
 		
 		private int GetHitCount(Directory dir, Term term)
 		{
 			IndexSearcher searcher = new IndexSearcher(dir);
-			int hitCount = searcher.Search(new TermQuery(term)).Length();
+			int hitCount = searcher.Search(new TermQuery(term), null, 1000).totalHits;
 			searcher.Close();
 			return hitCount;
 		}
@@ -472,13 +408,13 @@
 				bool autoCommit = (0 == pass);
 				
 				// First build up a starting index:
-				RAMDirectory startDir = new RAMDirectory();
+				MockRAMDirectory startDir = new MockRAMDirectory();
 				IndexWriter writer = new IndexWriter(startDir, autoCommit, new WhitespaceAnalyzer(), true);
 				for (int i = 0; i < 157; i++)
 				{
 					Document d = new Document();
-					d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.UN_TOKENIZED));
-					d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.TOKENIZED));
+					d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
+					d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.ANALYZED));
 					writer.AddDocument(d);
 				}
 				writer.Close();
@@ -494,6 +430,7 @@
 				while (!done)
 				{
 					MockRAMDirectory dir = new MockRAMDirectory(startDir);
+                    dir.SetPreventDoubleWrite(false);
 					IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
 					
 					modifier.SetMaxBufferedDocs(1000); // use flush or close
@@ -559,8 +496,8 @@
 									if (updates)
 									{
 										Document d = new Document();
-										d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.UN_TOKENIZED));
-										d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.TOKENIZED));
+										d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
+										d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED));
 										modifier.UpdateDocument(new Term("id", System.Convert.ToString(docId)), d);
 									}
 									else
@@ -583,7 +520,6 @@
 						{
 							if (debug)
 							{
-								//UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
 								System.Console.Out.WriteLine("  hit IOException: " + e);
 								System.Console.Out.WriteLine(e.StackTrace);
 							}
@@ -595,43 +531,12 @@
 							}
 						}
 						
-						// Whether we succeeded or failed, check that all
-						// un-referenced files were in fact deleted (ie,
-						// we did not create garbage). Just create a
-						// new IndexFileDeleter, have it delete
-						// unreferenced files, then verify that in fact
-						// no files were deleted:
-						System.String[] startFiles = dir.List();
-						SegmentInfos infos = new SegmentInfos();
-						infos.Read(dir);
-						new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
-						System.String[] endFiles = dir.List();
-						
-						//UPGRADE_TODO: Method 'java.util.Arrays.sort' was converted to 'System.Array.Sort' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javautilArrayssort_javalangObject[]'"
-						System.Array.Sort(startFiles);
-						//UPGRADE_TODO: Method 'java.util.Arrays.sort' was converted to 'System.Array.Sort' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javautilArrayssort_javalangObject[]'"
-						System.Array.Sort(endFiles);
-						
-						// for(int i=0;i<startFiles.length;i++) {
-						// System.out.println(" startFiles: " + i + ": " + startFiles[i]);
-						// }
-						
-						if (!SupportClass.Compare.CompareStringArrays(startFiles, endFiles))
-						{
-							System.String successStr;
-							if (success)
-							{
-								successStr = "success";
-							}
-							else
-							{
-								successStr = "IOException";
-								System.Console.Error.WriteLine(err.StackTrace);
-							}
-							Assert.Fail("reader.close() failed to delete unreferenced files after " + successStr + " (" + diskFree + " bytes): before delete:\n    " + ArrayToString(startFiles) + "\n  after delete:\n    " + ArrayToString(endFiles));
-						}
-						
-						// Finally, verify index is not corrupt, and, if
+
+                        // If the Close() succeeded, make sure there are no unreferenced files
+                        if (success)
+                            TestIndexWriter.AssertNoUnreferencedFiles(dir, "after writer.Close");
+
+                        // Finally, verify index is not corrupt, and, if
 						// we succeeded, we see all docs changed, and if
 						// we failed, we see either all docs or no docs
 						// changed (transactional semantics):
@@ -643,23 +548,21 @@
 						catch (System.IO.IOException e)
 						{
 							System.Console.Error.WriteLine(e.StackTrace);
-							//UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
 							Assert.Fail(testName + ":exception when creating IndexReader after disk full during close: " + e);
 						}
 						
 						IndexSearcher searcher = new IndexSearcher(newReader);
-						Hits hits = null;
+						ScoreDoc[] hits = null;
 						try
 						{
-							hits = searcher.Search(new TermQuery(searchTerm));
+							hits = searcher.Search(new TermQuery(searchTerm), null, 1000).scoreDocs;
 						}
 						catch (System.IO.IOException e)
 						{
 							System.Console.Error.WriteLine(e.StackTrace);
-							//UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
 							Assert.Fail(testName + ": exception when searching: " + e);
 						}
-						int result2 = hits.Length();
+						int result2 = hits.Length;
 						if (success)
 						{
 							if (x == 0 && result2 != END_COUNT)
@@ -733,23 +636,18 @@
 				for (int i = 0; i < keywords.Length; i++)
 				{
 					Document doc = new Document();
-					doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.UN_TOKENIZED));
+					doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
 					doc.Add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO));
-					doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.TOKENIZED));
-					doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.TOKENIZED));
+					doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED));
+					doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED));
 					modifier.AddDocument(doc);
 				}
 				// flush (and commit if ac)
-				
-				modifier.Optimize();
-				
-				// commit if !ac
-				
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
-				// one of the two files hits
+
+                modifier.Optimize();
+                modifier.Commit();
+
+                // one of the two files hits
 				
 				Term term = new Term("city", "Amsterdam");
 				int hitCount = GetHitCount(dir, term);
@@ -757,12 +655,6 @@
 				
 				// open the writer again (closed above)
 				
-				if (!autoCommit)
-				{
-					modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
-					modifier.SetUseCompoundFile(true);
-				}
-				
 				// delete the doc
 				// max buf del terms is two, so this is buffered
 				
@@ -774,7 +666,7 @@
 				Document doc2 = new Document();
 				modifier.AddDocument(doc2);
 				
-				// flush the changes, the buffered deletes, and the new doc
+				// commit the changes, the buffered deletes, and the new doc
 				
 				// The failure object will fail on the first write after the del
 				// file gets created when processing the buffered delete
@@ -790,7 +682,7 @@
 				bool failed = false;
 				try
 				{
-					modifier.Flush();
+					modifier.Commit();
 				}
 				catch (System.IO.IOException)
 				{
@@ -799,32 +691,18 @@
 				
 				Assert.IsTrue(failed);
 				
-				// The flush above failed, so we need to retry it (which will
+				// The commit above failed, so we need to retry it (which will
 				// succeed, because the failure is a one-shot)
-				
-				if (!autoCommit)
-				{
-					modifier.Close();
-				}
-				else
-				{
-					modifier.Flush();
-				}
+
+                modifier.Commit();
 				
 				hitCount = GetHitCount(dir, term);
 				
-				// If the delete was not cleared then hit count will
-				// be 0.  With autoCommit=false, we hit the exception
-				// on creating the compound file, so the delete was
-				// flushed successfully.
-				Assert.AreEqual(autoCommit?1:0, hitCount);
-				
-				if (autoCommit)
-				{
-					modifier.Close();
-				}
+				// Make sure the delete was successfully flushed
+				Assert.AreEqual(0, hitCount);
 				
-				dir.Close();
+                modifier.Close();
+                dir.Close();
 			}
 		}
 		
@@ -855,10 +733,10 @@
 				for (int i = 0; i < keywords.Length; i++)
 				{
 					Document doc = new Document();
-					doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.UN_TOKENIZED));
+					doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
 					doc.Add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO));
-					doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.TOKENIZED));
-					doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.TOKENIZED));
+					doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED));
+					doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED));
 					try
 					{
 						modifier.AddDocument(doc);

Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterExceptions.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterExceptions.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterExceptions.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterExceptions.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using NUnit.Framework;
+
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using _TestUtil = Lucene.Net.Util._TestUtil;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
+using Directory = Lucene.Net.Store.Directory;
+using Analyzer = Lucene.Net.Analysis.Analyzer;
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using CloseableThreadLocal = Lucene.Net.Util.CloseableThreadLocal;
+
+namespace Lucene.Net.Index
+{
+    [TestFixture]
+    public class TestIndexWriterExceptions : LuceneTestCase
+    {
+
+        private const bool DEBUG = false;
+
+        private class IndexerThread : SupportClass.ThreadClass
+        {
+            private TestIndexWriterExceptions enclosingInstance;
+            internal IndexWriter writer;
+
+            internal readonly System.Random r = new System.Random(47);
+            internal System.Exception failure;
+
+            public IndexerThread(int i, IndexWriter writer, TestIndexWriterExceptions enclosingInstance)
+                : base("Indexer " + i)
+            {
+                this.writer = writer;
+                this.enclosingInstance = enclosingInstance;
+            }
+
+            override public void Run()
+            {
+
+                Document doc = new Document();
+
+                doc.Add(new Field("content1", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.ANALYZED));
+                doc.Add(new Field("content6", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+                doc.Add(new Field("content2", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NOT_ANALYZED));
+                doc.Add(new Field("content3", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NO));
+
+                doc.Add(new Field("content4", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED));
+                doc.Add(new Field("content5", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED));
+
+                doc.Add(new Field("content7", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+
+                Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
+                doc.Add(idField);
+
+                System.DateTime stopTime = System.DateTime.Now.AddSeconds(3);
+
+                while (System.DateTime.Now < stopTime)
+                {
+                    enclosingInstance.doFail.Set(this);
+                    string id = "" + r.Next(50);
+                    idField.SetValue(id);
+                    Term idTerm = new Term("id", id);
+                    try
+                    {
+                        writer.UpdateDocument(idTerm, doc);
+                    }
+                    catch (System.Exception re)
+                    {
+                        if (DEBUG)
+                        {
+                            System.Console.Out.WriteLine("EXC: ");
+                            System.Console.Out.WriteLine(re.StackTrace);
+                        }
+                        try
+                        {
+                            _TestUtil.CheckIndex(writer.GetDirectory());
+                        }
+                        catch (System.IO.IOException ioe)
+                        {
+                            System.Console.Out.WriteLine(System.Threading.Thread.CurrentThread.Name + ": unexpected exception1");
+                            System.Console.Out.WriteLine(ioe.StackTrace);
+                            failure = ioe;
+                            break;
+                        }
+                        // this, in Java, was catch Throwable, and the catch above (at the same nesting level)
+                        // was catch RuntimeException... as all exceptions in C# are unchecked, these both come
+                        // down to System.Exception
+                        /*
+                    } catch (System.Exception t) {
+                      System.Console.Out.WriteLine(System.Threading.Thread.CurrentThread.Name + ": unexpected exception2");
+                      System.Console.Out.WriteLine(t.StackTrace);
+                      failure = t;
+                      break;
+                         */
+                    }
+
+                    enclosingInstance.doFail.Set(null);
+
+                    // After a possible exception (above) I should be able
+                    // to add a new document without hitting an
+                    // exception:
+                    try
+                    {
+                        writer.UpdateDocument(idTerm, doc);
+                    }
+                    catch (System.Exception t)
+                    {
+                        System.Console.Out.WriteLine(System.Threading.Thread.CurrentThread.Name + ": unexpected exception3");
+                        System.Console.Out.WriteLine(t.StackTrace);
+                        failure = t;
+                        break;
+                    }
+                }
+            }
+        }
+
+        CloseableThreadLocal doFail = new CloseableThreadLocal();
+
+        public class MockIndexWriter : IndexWriter
+        {
+            private TestIndexWriterExceptions enclosingInstance;
+
+            internal System.Random r = new System.Random(17);
+
+            public MockIndexWriter(Directory dir, Analyzer a, bool create, MaxFieldLength mfl, TestIndexWriterExceptions enclosingInstance)
+                : base(dir, a, create, mfl)
+            {
+                this.enclosingInstance = enclosingInstance;
+            }
+
+            protected override bool TestPoint(string name)
+            {
+                if (enclosingInstance.doFail.Get() != null && !name.Equals("startDoFlush") && r.Next(20) == 17)
+                {
+                    if (DEBUG)
+                    {
+                        System.Console.Out.WriteLine(System.Threading.Thread.CurrentThread.Name + ": NOW FAIL: " + name);
+                    }
+                    throw new System.Exception(System.Threading.Thread.CurrentThread.Name + ": intentionally failing at " + name);
+                }
+                return true;
+            }
+        }
+
+        [Test]
+        public void TestRandomExceptions()
+        {
+            MockRAMDirectory dir = new MockRAMDirectory();
+
+            MockIndexWriter writer = new MockIndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, this);
+            ((ConcurrentMergeScheduler)writer.GetMergeScheduler()).SetSuppressExceptions_ForNUnitTest();
+            //writer.setMaxBufferedDocs(10);
+            writer.SetRAMBufferSizeMB(0.1);
+
+            if (DEBUG)
+                writer.SetInfoStream(System.Console.Out);
+
+            IndexerThread thread = new IndexerThread(0, writer, this);
+            thread.Run();
+            if (thread.failure != null)
+            {
+                System.Console.Out.WriteLine(thread.failure.StackTrace);
+                Assert.Fail("thread " + thread.Name + ": hit unexpected failure");
+            }
+
+            writer.Commit();
+
+            try
+            {
+                writer.Close();
+            }
+            catch (System.Exception t)
+            {
+                System.Console.Out.WriteLine("exception during close:");
+                System.Console.Out.WriteLine(t.StackTrace);
+                writer.Rollback();
+            }
+
+            // Confirm that when doc hits exception partway through tokenization, it's deleted:
+            IndexReader r2 = IndexReader.Open(dir);
+            int count = r2.DocFreq(new Term("content4", "aaa"));
+            int count2 = r2.DocFreq(new Term("content4", "ddd"));
+            Assert.AreEqual(count, count2);
+            r2.Close();
+
+            _TestUtil.CheckIndex(dir);
+        }
+
+        [Test]
+        public void TestRandomExceptionsThreads()
+        {
+
+            MockRAMDirectory dir = new MockRAMDirectory();
+            MockIndexWriter writer = new MockIndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, this);
+            ((ConcurrentMergeScheduler)writer.GetMergeScheduler()).SetSuppressExceptions_ForNUnitTest();
+            //writer.setMaxBufferedDocs(10);
+            writer.SetRAMBufferSizeMB(0.2);
+
+            if (DEBUG)
+                writer.SetInfoStream(System.Console.Out);
+
+            int NUM_THREADS = 4;
+
+            IndexerThread[] threads = new IndexerThread[NUM_THREADS];
+            for (int i = 0; i < NUM_THREADS; i++)
+            {
+                threads[i] = new IndexerThread(i, writer, this);
+                threads[i].Start();
+            }
+
+            for (int i = 0; i < NUM_THREADS; i++)
+                threads[i].Join();
+
+            for (int i = 0; i < NUM_THREADS; i++)
+                if (threads[i].failure != null)
+                    Assert.Fail("thread " + threads[i].Name + ": hit unexpected failure");
+
+            writer.Commit();
+
+            try
+            {
+                writer.Close();
+            }
+            catch (System.Exception t)
+            {
+                System.Console.Out.WriteLine("exception during close:");
+                System.Console.Out.WriteLine(t.StackTrace);
+                writer.Rollback();
+            }
+
+            // Confirm that when doc hits exception partway through tokenization, it's deleted:
+            IndexReader r2 = IndexReader.Open(dir);
+            int count = r2.DocFreq(new Term("content4", "aaa"));
+            int count2 = r2.DocFreq(new Term("content4", "ddd"));
+            Assert.AreEqual(count, count2);
+            r2.Close();
+
+            _TestUtil.CheckIndex(dir);
+        }
+    }
+}

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterLockRelease.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterLockRelease.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterLockRelease.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterLockRelease.cs Wed Jul 29 18:04:12 2009
@@ -28,8 +28,6 @@
 	/// exist).
 	/// 
 	/// </summary>
-	/// <author>  mbogosian
-	/// </author>
 	/// <version>  $Id$
 	/// </version>
 	
@@ -130,13 +128,13 @@
 			
 			try
 			{
-				im = new IndexWriter(this.__test_dir, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), false);
+				im = new IndexWriter(this.__test_dir, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
 			}
 			catch (System.IO.FileNotFoundException)
 			{
 				try
 				{
-					im = new IndexWriter(this.__test_dir, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), false);
+                    im = new IndexWriter(this.__test_dir, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
 				}
 				catch (System.IO.FileNotFoundException)
 				{
@@ -144,4 +142,4 @@
 			}
 		}
 	}
-}
+}
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMergePolicy.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterMergePolicy.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMergePolicy.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMergePolicy.cs Wed Jul 29 18:04:12 2009
@@ -39,8 +39,8 @@
 		public virtual void  TestNormalCase()
 		{
 			Directory dir = new RAMDirectory();
-			
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			writer.SetMaxBufferedDocs(10);
 			writer.SetMergeFactor(10);
 			writer.SetMergePolicy(new LogDocMergePolicy());
@@ -59,8 +59,8 @@
 		public virtual void  TestNoOverMerge()
 		{
 			Directory dir = new RAMDirectory();
-			
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			writer.SetMaxBufferedDocs(10);
 			writer.SetMergeFactor(10);
 			writer.SetMergePolicy(new LogDocMergePolicy());
@@ -85,8 +85,8 @@
 		public virtual void  TestForceFlush()
 		{
 			Directory dir = new RAMDirectory();
-			
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			writer.SetMaxBufferedDocs(10);
 			writer.SetMergeFactor(10);
 			LogDocMergePolicy mp = new LogDocMergePolicy();
@@ -97,8 +97,8 @@
 			{
 				AddDoc(writer);
 				writer.Close();
-				
-				writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+
+                writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
 				writer.SetMaxBufferedDocs(10);
 				writer.SetMergeFactor(10);
 				mp.SetMinMergeDocs(100);
@@ -114,8 +114,8 @@
 		public virtual void  TestMergeFactorChange()
 		{
 			Directory dir = new RAMDirectory();
-			
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			writer.SetMaxBufferedDocs(10);
 			writer.SetMergeFactor(100);
 			writer.SetMergePolicy(new LogDocMergePolicy());
@@ -144,8 +144,8 @@
 		public virtual void  TestMaxBufferedDocsChange()
 		{
 			Directory dir = new RAMDirectory();
-			
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			writer.SetMaxBufferedDocs(101);
 			writer.SetMergeFactor(101);
 			writer.SetMergePolicy(new LogDocMergePolicy());
@@ -160,8 +160,8 @@
 					CheckInvariants(writer);
 				}
 				writer.Close();
-				
-				writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+
+                writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 				writer.SetMaxBufferedDocs(101);
 				writer.SetMergeFactor(101);
 				writer.SetMergePolicy(new LogDocMergePolicy());
@@ -193,7 +193,7 @@
 		{
 			Directory dir = new RAMDirectory();
 			
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+			IndexWriter writer = new IndexWriter(dir, true, new WhitespaceAnalyzer(), true);
 			writer.SetMergePolicy(new LogDocMergePolicy());
 			writer.SetMaxBufferedDocs(10);
 			writer.SetMergeFactor(100);
@@ -209,7 +209,7 @@
 			reader.DeleteDocuments(new Term("content", "aaa"));
 			reader.Close();
 			
-			writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+			writer = new IndexWriter(dir, true, new WhitespaceAnalyzer(), false);
 			writer.SetMergePolicy(new LogDocMergePolicy());
 			writer.SetMaxBufferedDocs(10);
 			writer.SetMergeFactor(5);
@@ -228,7 +228,7 @@
 		private void  AddDoc(IndexWriter writer)
 		{
 			Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
-			doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.TOKENIZED));
+			doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
 			writer.AddDocument(doc);
 		}
 		

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMerging.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterMerging.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMerging.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMerging.cs Wed Jul 29 18:04:12 2009
@@ -62,7 +62,7 @@
 
 			Directory merged = new MockRAMDirectory();
 			
-			IndexWriter writer = new IndexWriter(merged, new StandardAnalyzer(), true);
+			IndexWriter writer = new IndexWriter(merged, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			writer.SetMergeFactor(2);
 			
 			writer.AddIndexes(new Directory[]{indexA, indexB});
@@ -97,15 +97,15 @@
 		
 		private void  FillIndex(Directory dir, int start, int numDocs)
 		{
-			
-			IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true);
+
+            IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			writer.SetMergeFactor(2);
 			writer.SetMaxBufferedDocs(2);
 			
 			for (int i = start; i < (start + numDocs); i++)
 			{
 				Lucene.Net.Documents.Document temp = new Lucene.Net.Documents.Document();
-				temp.Add(new Field("count", ("" + i), Field.Store.YES, Field.Index.UN_TOKENIZED));
+				temp.Add(new Field("count", ("" + i), Field.Store.YES, Field.Index.NOT_ANALYZED));
 				
 				writer.AddDocument(temp);
 			}

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyBug.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestLazyBug.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyBug.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyBug.cs Wed Jul 29 18:04:12 2009
@@ -69,7 +69,7 @@
 			{
 				System.Random r = new System.Random((System.Int32) (BASE_SEED + 42));
 				Analyzer analyzer = new SimpleAnalyzer();
-				IndexWriter writer = new IndexWriter(dir, analyzer, true);
+                IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
 				
 				writer.SetUseCompoundFile(false);
 				
@@ -78,7 +78,7 @@
 					Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
 					for (int f = 1; f <= NUM_FIELDS; f++)
 					{
-						doc.Add(new Field("f" + f, data[f % data.Length] + '#' + data[r.Next(data.Length)], Field.Store.YES, Field.Index.TOKENIZED));
+						doc.Add(new Field("f" + f, data[f % data.Length] + '#' + data[r.Next(data.Length)], Field.Store.YES, Field.Index.ANALYZED));
 					}
 					writer.AddDocument(doc);
 				}

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyProxSkipping.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestLazyProxSkipping.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyProxSkipping.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyProxSkipping.cs Wed Jul 29 18:04:12 2009
@@ -14,7 +14,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 using System;
 
 using NUnit.Framework;
@@ -25,7 +24,7 @@
 using IndexInput = Lucene.Net.Store.IndexInput;
 using RAMDirectory = Lucene.Net.Store.RAMDirectory;
 using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
-using Hits = Lucene.Net.Search.Hits;
+using ScoreDoc = Lucene.Net.Search.ScoreDoc;
 using IndexSearcher = Lucene.Net.Search.IndexSearcher;
 using PhraseQuery = Lucene.Net.Search.PhraseQuery;
 using Searcher = Lucene.Net.Search.Searcher;
@@ -53,7 +52,7 @@
 			int numDocs = 500;
 			
 			Directory directory = new RAMDirectory();
-			IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true);
+			IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			writer.SetMaxBufferedDocs(10);
 			for (int i = 0; i < numDocs; i++)
 			{
@@ -75,7 +74,7 @@
 					content = this.term3 + " " + this.term2;
 				}
 				
-				doc.Add(new Field(this.field, content, Field.Store.YES, Field.Index.TOKENIZED));
+				doc.Add(new Field(this.field, content, Field.Store.YES, Field.Index.ANALYZED));
 				writer.AddDocument(doc);
 			}
 			
@@ -92,22 +91,22 @@
 			this.searcher = new IndexSearcher(reader);
 		}
 		
-		private Hits Search()
+		private ScoreDoc[] Search()
 		{
 			// create PhraseQuery "term1 term2" and search
 			PhraseQuery pq = new PhraseQuery();
 			pq.Add(new Term(this.field, this.term1));
 			pq.Add(new Term(this.field, this.term2));
-			return this.searcher.Search(pq);
+			return this.searcher.Search(pq, null, 1000).scoreDocs;
 		}
 		
 		private void  PerformTest(int numHits)
 		{
 			CreateIndex(numHits);
 			this.seeksCounter = 0;
-			Hits hits = Search();
+			ScoreDoc[] hits = Search();
 			// verify that the right number of docs was found
-			Assert.AreEqual(numHits, hits.Length());
+			Assert.AreEqual(numHits, hits.Length);
 			
 			// check if the number of calls of seek() does not exceed the number of hits
 			Assert.IsTrue(this.seeksCounter <= numHits + 1);
@@ -125,11 +124,11 @@
 		public virtual void  TestSeek()
 		{
 			Directory directory = new RAMDirectory();
-			IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true);
+            IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			for (int i = 0; i < 10; i++)
 			{
 				Document doc = new Document();
-				doc.Add(new Field(this.field, "a b", Field.Store.YES, Field.Index.TOKENIZED));
+				doc.Add(new Field(this.field, "a b", Field.Store.YES, Field.Index.ANALYZED));
 				writer.AddDocument(doc);
 			}
 			

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiLevelSkipList.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestMultiLevelSkipList.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiLevelSkipList.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiLevelSkipList.cs Wed Jul 29 18:04:12 2009
@@ -47,12 +47,12 @@
 		public virtual void  TestSimpleSkip()
 		{
 			RAMDirectory dir = new RAMDirectory();
-			IndexWriter writer = new IndexWriter(dir, new PayloadAnalyzer(), true);
+            IndexWriter writer = new IndexWriter(dir, new PayloadAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			Term term = new Term("test", "a");
 			for (int i = 0; i < 5000; i++)
 			{
 				Document d1 = new Document();
-				d1.Add(new Field(term.Field(), term.Text(), Field.Store.NO, Field.Index.TOKENIZED));
+				d1.Add(new Field(term.Field(), term.Text(), Field.Store.NO, Field.Index.ANALYZED));
 				writer.AddDocument(d1);
 			}
 			writer.Flush();
@@ -110,14 +110,15 @@
 			{
 			}
 			
-			public override Token Next()
+			public override Token Next(Token reusableToken)
 			{
-				Token t = input.Next();
-				if (t != null)
+                System.Diagnostics.Debug.Assert(reusableToken != null);
+				Token nextToken = input.Next(reusableToken);
+				if (nextToken != null)
 				{
-					t.SetPayload(new Payload(new byte[]{(byte) count++}));
+					nextToken.SetPayload(new Payload(new byte[]{(byte) count++}));
 				}
-				return t;
+				return nextToken;
 			}
 		}
 		

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiSegmentReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestMultiSegmentReader.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiSegmentReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiSegmentReader.cs Wed Jul 29 18:04:12 2009
@@ -52,8 +52,8 @@
 			doc2 = new Document();
 			DocHelper.SetupDoc(doc1);
 			DocHelper.SetupDoc(doc2);
-			SegmentInfo info1 = DocHelper.WriteDoc(dir, doc1);
-			SegmentInfo info2 = DocHelper.WriteDoc(dir, doc2);
+			DocHelper.WriteDoc(dir, doc1);
+			DocHelper.WriteDoc(dir, doc2);
 			sis = new SegmentInfos();
 			sis.Read(dir);
 		}
@@ -113,7 +113,7 @@
 			if (reader is MultiReader)
 			// MultiReader does not "own" the directory so it does
 			// not write the changes to sis on commit:
-				sis.Write(dir);
+				sis.Commit(dir);
 			
 			sis.Read(dir);
 			reader = OpenReader();
@@ -126,7 +126,7 @@
 			if (reader is MultiReader)
 			// MultiReader does not "own" the directory so it does
 			// not write the changes to sis on commit:
-				sis.Write(dir);
+				sis.Commit(dir);
 			sis.Read(dir);
 			reader = OpenReader();
 			Assert.AreEqual(1, reader.NumDocs());
@@ -168,9 +168,9 @@
 		
 		private void  AddDoc(RAMDirectory ramDir1, System.String s, bool create)
 		{
-			IndexWriter iw = new IndexWriter(ramDir1, new StandardAnalyzer(), create);
+			IndexWriter iw = new IndexWriter(ramDir1, new StandardAnalyzer(), create, IndexWriter.MaxFieldLength.LIMITED);
 			Document doc = new Document();
-			doc.Add(new Field("body", s, Field.Store.YES, Field.Index.TOKENIZED));
+			doc.Add(new Field("body", s, Field.Store.YES, Field.Index.ANALYZED));
 			iw.AddDocument(doc);
 			iw.Close();
 		}

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestNorms.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestNorms.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestNorms.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestNorms.cs Wed Jul 29 18:04:12 2009
@@ -92,7 +92,7 @@
 		/// Including optimize. 
 		/// </summary>
 		[Test]
-		public virtual void  _TestNorms()
+		public virtual void  TestNorms_Renamed()
 		{
 			// tmp dir
 			System.String tempDir = System.IO.Path.GetTempPath();
@@ -131,7 +131,7 @@
 			Directory dir3 = FSDirectory.GetDirectory(indexDir3);
 			
 			CreateIndex(dir3);
-			IndexWriter iw = new IndexWriter(dir3, anlzr, false);
+            IndexWriter iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
 			iw.SetMaxBufferedDocs(5);
 			iw.SetMergeFactor(3);
 			iw.AddIndexes(new Directory[]{dir1, dir2});
@@ -148,7 +148,7 @@
 			DoTestNorms(dir3);
 			
 			// now with optimize
-			iw = new IndexWriter(dir3, anlzr, false);
+            iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
 			iw.SetMaxBufferedDocs(5);
 			iw.SetMergeFactor(3);
 			iw.Optimize();
@@ -177,7 +177,7 @@
 		
 		private void  CreateIndex(Directory dir)
 		{
-			IndexWriter iw = new IndexWriter(dir, anlzr, true);
+            IndexWriter iw = new IndexWriter(dir, anlzr, true, IndexWriter.MaxFieldLength.LIMITED);
 			iw.SetMaxBufferedDocs(5);
 			iw.SetMergeFactor(3);
 			iw.SetSimilarity(similarityOne);
@@ -226,7 +226,7 @@
 		
 		private void  AddDocs(Directory dir, int ndocs, bool compound)
 		{
-			IndexWriter iw = new IndexWriter(dir, anlzr, false);
+            IndexWriter iw = new IndexWriter(dir, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
 			iw.SetMaxBufferedDocs(5);
 			iw.SetMergeFactor(3);
 			iw.SetSimilarity(similarityOne);
@@ -245,7 +245,7 @@
 			float boost = NextNorm();
 			for (int i = 0; i < 10; i++)
 			{
-				Field f = new Field("f" + i, "v" + i, Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.UN_TOKENIZED);
+				Field f = new Field("f" + i, "v" + i, Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.NOT_ANALYZED);
 				f.SetBoost(boost);
 				d.Add(f);
 			}

Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestOmitTf.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestOmitTf.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestOmitTf.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestOmitTf.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,373 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using NUnit.Framework;
+
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using _TestUtil = Lucene.Net.Util._TestUtil;
+using Analyzer = Lucene.Net.Analysis.Analyzer;
+using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using BooleanQuery = Lucene.Net.Search.BooleanQuery;
+using HitCollector = Lucene.Net.Search.HitCollector;
+using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+using Searcher = Lucene.Net.Search.Searcher;
+using Similarity = Lucene.Net.Search.Similarity;
+using TermQuery = Lucene.Net.Search.TermQuery;
+using Occur = Lucene.Net.Search.BooleanClause.Occur;
+using Directory = Lucene.Net.Store.Directory;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
+
+namespace Lucene.Net.Index
+{
+    [TestFixture]
+    public class TestOmitTf : LuceneTestCase
+    {
+
+        public class SimpleSimilarity : Similarity
+        {
+            override public float LengthNorm(string field, int numTerms) { return 1.0f; }
+            override public float QueryNorm(float sumOfSquaredWeights) { return 1.0f; }
+            override public float Tf(float freq) { return freq; }
+            override public float SloppyFreq(int distance) { return 2.0f; }
+            override public float Idf(System.Collections.ICollection terms, Searcher searcher) { return 1.0f; }
+            override public float Idf(int docFreq, int numDocs) { return 1.0f; }
+            override public float Coord(int overlap, int maxOverlap) { return 1.0f; }
+        }
+
+        // Tests whether the DocumentWriter correctly enable the
+        // omitTf bit in the FieldInfo
+        [Test]
+        public void TestOmitTf_Renamed()
+        {
+            Directory ram = new MockRAMDirectory();
+            Analyzer analyzer = new StandardAnalyzer();
+            IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+            Document d = new Document();
+
+            // this field will have Tf
+            Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
+            d.Add(f1);
+
+            // this field will NOT have Tf
+            Field f2 = new Field("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
+            f2.SetOmitTf(true);
+            d.Add(f2);
+
+            writer.AddDocument(d);
+            writer.Optimize();
+            // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger
+            // keep things constant
+            d = new Document();
+
+            // Reverese
+            f1.SetOmitTf(true);
+            d.Add(f1);
+
+            f2.SetOmitTf(false);
+            d.Add(f2);
+
+            writer.AddDocument(d);
+            // force merge
+            writer.Optimize();
+            // flush
+            writer.Close();
+            _TestUtil.CheckIndex(ram);
+
+            // only one segment in the index, so we can cast to SegmentReader
+            SegmentReader reader = (SegmentReader)IndexReader.Open(ram);
+            FieldInfos fi = reader.FieldInfos();
+            Assert.IsTrue(fi.FieldInfo("f1").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+            Assert.IsTrue(fi.FieldInfo("f2").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+
+            reader.Close();
+            ram.Close();
+        }
+
+        // Tests whether merging of docs that have different
+        // omitTf for the same field works
+        [Test]
+        public void TestMixedMerge()
+        {
+            Directory ram = new MockRAMDirectory();
+            Analyzer analyzer = new StandardAnalyzer();
+            IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+            writer.SetMaxBufferedDocs(3);
+            writer.SetMergeFactor(2);
+            Document d = new Document();
+
+            // this field will have Tf
+            Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
+            d.Add(f1);
+
+            // this field will NOT have Tf
+            Field f2 = new Field("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
+            f2.SetOmitTf(true);
+            d.Add(f2);
+
+            for (int i = 0; i < 30; i++)
+                writer.AddDocument(d);
+
+            // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger
+            // keep things constant
+            d = new Document();
+
+            // Reverese
+            f1.SetOmitTf(true);
+            d.Add(f1);
+
+            f2.SetOmitTf(false);
+            d.Add(f2);
+
+            for (int i = 0; i < 30; i++)
+                writer.AddDocument(d);
+
+            // force merge
+            writer.Optimize();
+            // flush
+            writer.Close();
+
+            _TestUtil.CheckIndex(ram);
+
+            // only one segment in the index, so we can cast to SegmentReader
+            SegmentReader reader = (SegmentReader)IndexReader.Open(ram);
+            FieldInfos fi = reader.FieldInfos();
+            Assert.IsTrue(fi.FieldInfo("f1").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+            Assert.IsTrue(fi.FieldInfo("f2").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+
+            reader.Close();
+            ram.Close();
+        }
+
+        // Make sure first adding docs that do not omitTf for
+        // field X, then adding docs that do omitTf for that same
+        // field, 
+        [Test]
+        public void TestMixedRAM()
+        {
+            Directory ram = new MockRAMDirectory();
+            Analyzer analyzer = new StandardAnalyzer();
+            IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+            writer.SetMaxBufferedDocs(10);
+            writer.SetMergeFactor(2);
+            Document d = new Document();
+
+            // this field will have Tf
+            Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
+            d.Add(f1);
+
+            // this field will NOT have Tf
+            Field f2 = new Field("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
+            d.Add(f2);
+
+            for (int i = 0; i < 5; i++)
+                writer.AddDocument(d);
+
+            f2.SetOmitTf(true);
+
+            for (int i = 0; i < 20; i++)
+                writer.AddDocument(d);
+
+            // force merge
+            writer.Optimize();
+
+            // flush
+            writer.Close();
+
+            _TestUtil.CheckIndex(ram);
+
+            // only one segment in the index, so we can cast to SegmentReader
+            SegmentReader reader = (SegmentReader)IndexReader.Open(ram);
+            FieldInfos fi = reader.FieldInfos();
+            Assert.IsTrue(!fi.FieldInfo("f1").omitTf_ForNUnitTest, "OmitTf field bit should not be set.");
+            Assert.IsTrue(fi.FieldInfo("f2").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+
+            reader.Close();
+            ram.Close();
+        }
+
+        private void AssertNoPrx(Directory dir)
+        {
+            string[] files = dir.List();
+            for (int i = 0; i < files.Length; i++)
+                Assert.IsFalse(files[i].EndsWith(".prx"));
+        }
+
+        // Verifies no *.prx exists when all fields omit term freq:
+        [Test]
+        public void TestNoPrxFile()
+        {
+            Directory ram = new MockRAMDirectory();
+            Analyzer analyzer = new StandardAnalyzer();
+            IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+            writer.SetMaxBufferedDocs(3);
+            writer.SetMergeFactor(2);
+            writer.SetUseCompoundFile(false);
+            Document d = new Document();
+
+            Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
+            f1.SetOmitTf(true);
+            d.Add(f1);
+
+            for (int i = 0; i < 30; i++)
+                writer.AddDocument(d);
+
+            writer.Commit();
+
+            AssertNoPrx(ram);
+
+            // force merge
+            writer.Optimize();
+            // flush
+            writer.Close();
+
+            AssertNoPrx(ram);
+            _TestUtil.CheckIndex(ram);
+            ram.Close();
+        }
+
+        // Test scores with one field with Term Freqs and one without, otherwise with equal content 
+        [Test]
+        public void TestBasic()
+        {
+            Directory dir = new MockRAMDirectory();
+            Analyzer analyzer = new StandardAnalyzer();
+            IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+            writer.SetMergeFactor(2);
+            writer.SetMaxBufferedDocs(2);
+            writer.SetSimilarity(new SimpleSimilarity());
+
+
+            System.Text.StringBuilder sb = new System.Text.StringBuilder(265);
+            string term = "term";
+            for (int i = 0; i < 30; i++)
+            {
+                Document d = new Document();
+                sb.Append(term).Append(" ");
+                string content = sb.ToString();
+                Field noTf = new Field("noTf", content + (i % 2 == 0 ? "" : " notf"), Field.Store.NO, Field.Index.ANALYZED);
+                noTf.SetOmitTf(true);
+                d.Add(noTf);
+
+                Field tf = new Field("tf", content + (i % 2 == 0 ? " tf" : ""), Field.Store.NO, Field.Index.ANALYZED);
+                d.Add(tf);
+
+                writer.AddDocument(d);
+                //System.out.println(d);
+            }
+
+            writer.Optimize();
+            // flush
+            writer.Close();
+            _TestUtil.CheckIndex(dir);
+
+            /*
+             * Verify the index
+             */
+            Searcher searcher = new IndexSearcher(dir);
+            searcher.SetSimilarity(new SimpleSimilarity());
+
+            Term a = new Term("noTf", term);
+            Term b = new Term("tf", term);
+            Term c = new Term("noTf", "notf");
+            Term d2 = new Term("tf", "tf");
+            TermQuery q1 = new TermQuery(a);
+            TermQuery q2 = new TermQuery(b);
+            TermQuery q3 = new TermQuery(c);
+            TermQuery q4 = new TermQuery(d2);
+
+
+            searcher.Search(q1, new AnonymousCountingHitCollector1());
+            searcher.Search(q2, new AnonymousCountingHitCollector2());
+            searcher.Search(q3, new AnonymousCountingHitCollector3());
+            searcher.Search(q4, new AnonymousCountingHitCollector4());
+
+            BooleanQuery bq = new BooleanQuery();
+            bq.Add(q1, Occur.MUST);
+            bq.Add(q4, Occur.MUST);
+
+            searcher.Search(bq, new AnonymousCountingHitCollector5());
+            Assert.IsTrue(15 == CountingHitCollector.GetCount());
+
+            searcher.Close();
+            dir.Close();
+        }
+
+        public class CountingHitCollector : HitCollector
+        {
+            static int count = 0;
+            static int sum = 0;
+            internal CountingHitCollector() { count = 0; sum = 0; }
+            override public void Collect(int doc, float score)
+            {
+                count++;
+                sum += doc;  // use it to avoid any possibility of being optimized away
+            }
+
+            public static int GetCount() { return count; }
+            public static int GetSum() { return sum; }
+        }
+
+        public class AnonymousCountingHitCollector1 : CountingHitCollector
+        {
+            override public void Collect(int doc, float score)
+            {
+                //System.out.println("Q1: Doc=" + doc + " score=" + score);
+                Assert.IsTrue(score == 1.0f);
+                base.Collect(doc, score);
+            }
+        }
+        public class AnonymousCountingHitCollector2 : CountingHitCollector
+        {
+            override public void Collect(int doc, float score)
+            {
+                //System.out.println("Q1: Doc=" + doc + " score=" + score);
+                Assert.IsTrue(score == 1.0f+doc);
+                base.Collect(doc, score);
+            }
+        }
+        public class AnonymousCountingHitCollector3 : CountingHitCollector
+        {
+            override public void Collect(int doc, float score)
+            {
+                //System.out.println("Q1: Doc=" + doc + " score=" + score);
+                Assert.IsTrue(score == 1.0f);
+                Assert.IsFalse(doc % 2 == 0);
+                base.Collect(doc, score);
+            }
+        }
+        public class AnonymousCountingHitCollector4 : CountingHitCollector
+        {
+            override public void Collect(int doc, float score)
+            {
+                //System.out.println("Q1: Doc=" + doc + " score=" + score);
+                Assert.IsTrue(score == 1.0f);
+                Assert.IsTrue(doc % 2 == 0);
+                base.Collect(doc, score);
+            }
+        };
+        public class AnonymousCountingHitCollector5 : CountingHitCollector
+        {
+            override public void Collect(int doc, float score)
+            {
+                //System.out.println("BQ: Doc=" + doc + " score=" + score);
+                base.Collect(doc, score);
+            }
+        }
+    }
+}

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestParallelReader.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs Wed Jul 29 18:04:12 2009
@@ -27,7 +27,7 @@
 using RAMDirectory = Lucene.Net.Store.RAMDirectory;
 using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
 using BooleanQuery = Lucene.Net.Search.BooleanQuery;
-using Hits = Lucene.Net.Search.Hits;
+using ScoreDoc = Lucene.Net.Search.ScoreDoc;
 using IndexSearcher = Lucene.Net.Search.IndexSearcher;
 using Query = Lucene.Net.Search.Query;
 using Searcher = Lucene.Net.Search.Searcher;
@@ -78,12 +78,12 @@
 			ParallelReader pr = new ParallelReader();
 			pr.Add(IndexReader.Open(dir1));
 			pr.Add(IndexReader.Open(dir2));
-			System.Collections.ICollection fieldNames = pr.GetFieldNames(IndexReader.FieldOption.ALL);
+			System.Collections.Generic.ICollection<string> fieldNames = pr.GetFieldNames(IndexReader.FieldOption.ALL);
 			Assert.AreEqual(4, fieldNames.Count);
-			Assert.IsTrue(CollectionContains(fieldNames, "f1"));
-			Assert.IsTrue(CollectionContains(fieldNames, "f2"));
-			Assert.IsTrue(CollectionContains(fieldNames, "f3"));
-			Assert.IsTrue(CollectionContains(fieldNames, "f4"));
+			Assert.IsTrue(fieldNames.Contains("f1"));
+			Assert.IsTrue(fieldNames.Contains("f2"));
+			Assert.IsTrue(fieldNames.Contains("f3"));
+			Assert.IsTrue(fieldNames.Contains("f4"));
 		}
 		
 		[Test]
@@ -117,9 +117,9 @@
 			
 			// one document only:
 			Directory dir2 = new MockRAMDirectory();
-			IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true);
+            IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			Lucene.Net.Documents.Document d3 = new Lucene.Net.Documents.Document();
-			d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+			d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.ANALYZED));
 			w2.AddDocument(d3);
 			w2.Close();
 			
@@ -169,15 +169,15 @@
 			Directory dir2 = GetDir1();
 			
 			// add another document to ensure that the indexes are not optimized
-			IndexWriter modifier = new IndexWriter(dir1, new StandardAnalyzer());
+            IndexWriter modifier = new IndexWriter(dir1, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
 			Document d = new Document();
-			d.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+			d.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.ANALYZED));
 			modifier.AddDocument(d);
 			modifier.Close();
-			
-			modifier = new IndexWriter(dir2, new StandardAnalyzer());
+
+            modifier = new IndexWriter(dir2, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
 			d = new Document();
-			d.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.TOKENIZED));
+			d.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.ANALYZED));
 			modifier.AddDocument(d);
 			modifier.Close();
 			
@@ -187,8 +187,8 @@
 			pr.Add(IndexReader.Open(dir2));
 			Assert.IsFalse(pr.IsOptimized());
 			pr.Close();
-			
-			modifier = new IndexWriter(dir1, new StandardAnalyzer());
+
+            modifier = new IndexWriter(dir1, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
 			modifier.Optimize();
 			modifier.Close();
 			
@@ -198,9 +198,9 @@
 			// just one of the two indexes are optimized
 			Assert.IsFalse(pr.IsOptimized());
 			pr.Close();
-			
-			
-			modifier = new IndexWriter(dir2, new StandardAnalyzer());
+
+
+            modifier = new IndexWriter(dir2, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
 			modifier.Optimize();
 			modifier.Close();
 			
@@ -215,14 +215,14 @@
 		
 		private void  QueryTest(Query query)
 		{
-			Hits parallelHits = parallel.Search(query);
-			Hits singleHits = single.Search(query);
-			Assert.AreEqual(parallelHits.Length(), singleHits.Length());
-			for (int i = 0; i < parallelHits.Length(); i++)
+			ScoreDoc[] parallelHits = parallel.Search(query, null, 1000).scoreDocs;
+            ScoreDoc[] singleHits = single.Search(query, null, 1000).scoreDocs;
+			Assert.AreEqual(parallelHits.Length, singleHits.Length);
+			for (int i = 0; i < parallelHits.Length; i++)
 			{
-				Assert.AreEqual(parallelHits.Score(i), singleHits.Score(i), 0.001f);
-				Lucene.Net.Documents.Document docParallel = parallelHits.Doc(i);
-				Lucene.Net.Documents.Document docSingle = singleHits.Doc(i);
+				Assert.AreEqual(parallelHits[i].score, singleHits[i].score, 0.001f);
+				Lucene.Net.Documents.Document docParallel = parallel.Doc(parallelHits[i].doc);
+				Lucene.Net.Documents.Document docSingle = single.Doc(singleHits[i].doc);
 				Assert.AreEqual(docParallel.Get("f1"), docSingle.Get("f1"));
 				Assert.AreEqual(docParallel.Get("f2"), docSingle.Get("f2"));
 				Assert.AreEqual(docParallel.Get("f3"), docSingle.Get("f3"));
@@ -234,18 +234,18 @@
 		private Searcher Single()
 		{
 			Directory dir = new MockRAMDirectory();
-			IndexWriter w = new IndexWriter(dir, new StandardAnalyzer(), true);
+			IndexWriter w = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			Lucene.Net.Documents.Document d1 = new Lucene.Net.Documents.Document();
-			d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.TOKENIZED));
-			d1.Add(new Field("f2", "v1", Field.Store.YES, Field.Index.TOKENIZED));
-			d1.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.TOKENIZED));
-			d1.Add(new Field("f4", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+			d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.ANALYZED));
+			d1.Add(new Field("f2", "v1", Field.Store.YES, Field.Index.ANALYZED));
+			d1.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.ANALYZED));
+			d1.Add(new Field("f4", "v1", Field.Store.YES, Field.Index.ANALYZED));
 			w.AddDocument(d1);
 			Lucene.Net.Documents.Document d2 = new Lucene.Net.Documents.Document();
-			d2.Add(new Field("f1", "v2", Field.Store.YES, Field.Index.TOKENIZED));
-			d2.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.TOKENIZED));
-			d2.Add(new Field("f3", "v2", Field.Store.YES, Field.Index.TOKENIZED));
-			d2.Add(new Field("f4", "v2", Field.Store.YES, Field.Index.TOKENIZED));
+			d2.Add(new Field("f1", "v2", Field.Store.YES, Field.Index.ANALYZED));
+			d2.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.ANALYZED));
+			d2.Add(new Field("f3", "v2", Field.Store.YES, Field.Index.ANALYZED));
+			d2.Add(new Field("f4", "v2", Field.Store.YES, Field.Index.ANALYZED));
 			w.AddDocument(d2);
 			w.Close();
 			
@@ -266,14 +266,14 @@
 		private Directory GetDir1()
 		{
 			Directory dir1 = new MockRAMDirectory();
-			IndexWriter w1 = new IndexWriter(dir1, new StandardAnalyzer(), true);
+            IndexWriter w1 = new IndexWriter(dir1, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			Lucene.Net.Documents.Document d1 = new Lucene.Net.Documents.Document();
-			d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.TOKENIZED));
-			d1.Add(new Field("f2", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+			d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.ANALYZED));
+			d1.Add(new Field("f2", "v1", Field.Store.YES, Field.Index.ANALYZED));
 			w1.AddDocument(d1);
 			Lucene.Net.Documents.Document d2 = new Lucene.Net.Documents.Document();
-			d2.Add(new Field("f1", "v2", Field.Store.YES, Field.Index.TOKENIZED));
-			d2.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.TOKENIZED));
+			d2.Add(new Field("f1", "v2", Field.Store.YES, Field.Index.ANALYZED));
+			d2.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.ANALYZED));
 			w1.AddDocument(d2);
 			w1.Close();
 			return dir1;
@@ -282,28 +282,17 @@
 		private Directory GetDir2()
 		{
 			Directory dir2 = new RAMDirectory();
-			IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true);
+            IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			Lucene.Net.Documents.Document d3 = new Lucene.Net.Documents.Document();
-			d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.TOKENIZED));
-			d3.Add(new Field("f4", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+			d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.ANALYZED));
+			d3.Add(new Field("f4", "v1", Field.Store.YES, Field.Index.ANALYZED));
 			w2.AddDocument(d3);
 			Lucene.Net.Documents.Document d4 = new Lucene.Net.Documents.Document();
-			d4.Add(new Field("f3", "v2", Field.Store.YES, Field.Index.TOKENIZED));
-			d4.Add(new Field("f4", "v2", Field.Store.YES, Field.Index.TOKENIZED));
+			d4.Add(new Field("f3", "v2", Field.Store.YES, Field.Index.ANALYZED));
+			d4.Add(new Field("f4", "v2", Field.Store.YES, Field.Index.ANALYZED));
 			w2.AddDocument(d4);
 			w2.Close();
 			return dir2;
 		}
-
-		public static bool CollectionContains(System.Collections.ICollection col, System.String val)
-		{
-			for (System.Collections.IEnumerator iterator = col.GetEnumerator(); iterator.MoveNext(); )
-			{
-                System.String s = (System.String)iterator.Current;
-                if (s == val)
-					return true;
-			}
-			return false;
-		}
 	}
-}
+}
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestParallelTermEnum.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs Wed Jul 29 18:04:12 2009
@@ -44,22 +44,22 @@
 			Lucene.Net.Documents.Document doc;
 			
 			RAMDirectory rd1 = new RAMDirectory();
-			IndexWriter iw1 = new IndexWriter(rd1, new SimpleAnalyzer(), true);
+            IndexWriter iw1 = new IndexWriter(rd1, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			
 			doc = new Lucene.Net.Documents.Document();
-			doc.Add(new Field("field1", "the quick brown fox jumps", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.TOKENIZED));
-			doc.Add(new Field("field2", "the quick brown fox jumps", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.TOKENIZED));
-			doc.Add(new Field("field4", "", Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.TOKENIZED));
+			doc.Add(new Field("field1", "the quick brown fox jumps", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
+			doc.Add(new Field("field2", "the quick brown fox jumps", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
+			doc.Add(new Field("field4", "", Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.ANALYZED));
 			iw1.AddDocument(doc);
 			
 			iw1.Close();
 			RAMDirectory rd2 = new RAMDirectory();
-			IndexWriter iw2 = new IndexWriter(rd2, new SimpleAnalyzer(), true);
+            IndexWriter iw2 = new IndexWriter(rd2, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
 			
 			doc = new Lucene.Net.Documents.Document();
-			doc.Add(new Field("field0", "", Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.TOKENIZED));
-			doc.Add(new Field("field1", "the fox jumps over the lazy dog", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.TOKENIZED));
-			doc.Add(new Field("field3", "the fox jumps over the lazy dog", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.TOKENIZED));
+			doc.Add(new Field("field0", "", Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.ANALYZED));
+			doc.Add(new Field("field1", "the fox jumps over the lazy dog", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
+			doc.Add(new Field("field3", "the fox jumps over the lazy dog", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
 			iw2.AddDocument(doc);
 			
 			iw2.Close();

Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestPayloads.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs Wed Jul 29 18:04:12 2009
@@ -31,6 +31,7 @@
 using TokenStream = Lucene.Net.Analysis.TokenStream;
 using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
 using WhitespaceTokenizer = Lucene.Net.Analysis.WhitespaceTokenizer;
+using UnicodeUtil = Lucene.Net.Util.UnicodeUtil;
 
 namespace Lucene.Net.Index
 {
@@ -149,18 +150,18 @@
 		{
 			Directory ram = new RAMDirectory();
 			PayloadAnalyzer analyzer = new PayloadAnalyzer();
-			IndexWriter writer = new IndexWriter(ram, analyzer, true);
+            IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
 			Document d = new Document();
 			// this field won't have any payloads
-			d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.TOKENIZED));
+			d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.ANALYZED));
 			// this field will have payloads in all docs, however not for all term positions,
 			// so this field is used to check if the DocumentWriter correctly enables the payloads bit
 			// even if only some term positions have payloads
-			d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
-			d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
+			d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.ANALYZED));
+			d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.ANALYZED));
 			// this field is used to verify if the SegmentMerger enables payloads for a field if it has payloads 
 			// enabled in only some documents
-			d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.TOKENIZED));
+			d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.ANALYZED));
 			// only add payload data for field f2
 			analyzer.SetPayloadData("f2", 1, System.Text.Encoding.UTF8.GetBytes("somedata"), 0, 1);
 			writer.AddDocument(d);
@@ -177,12 +178,12 @@
 			
 			// now we add another document which has payloads for field f3 and verify if the SegmentMerger
 			// enabled payloads for that field
-			writer = new IndexWriter(ram, analyzer, true);
+            writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
 			d = new Document();
-			d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.TOKENIZED));
-			d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
-			d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
-			d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.TOKENIZED));
+			d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.ANALYZED));
+			d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.ANALYZED));
+			d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.ANALYZED));
+			d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.ANALYZED));
 			// add payload data for field f2 and f3
 			analyzer.SetPayloadData("f2", System.Text.Encoding.UTF8.GetBytes("somedata"), 0, 1);
 			analyzer.SetPayloadData("f3", System.Text.Encoding.UTF8.GetBytes("somedata"), 0, 3);
@@ -221,7 +222,7 @@
 		private void  PerformTest(Directory dir)
 		{
 			PayloadAnalyzer analyzer = new PayloadAnalyzer();
-			IndexWriter writer = new IndexWriter(dir, analyzer, true);
+            IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
 			
 			// should be in sync with value in TermInfosWriter
 			int skipInterval = 16;
@@ -245,7 +246,7 @@
 			byte[] payloadData = GenerateRandomData(payloadDataLength);
 			
 			Document d = new Document();
-			d.Add(new Field(fieldName, content, Field.Store.NO, Field.Index.TOKENIZED));
+			d.Add(new Field(fieldName, content, Field.Store.NO, Field.Index.ANALYZED));
 			// add the same document multiple times to have the same payload lengths for all
 			// occurrences within two consecutive skip intervals
 			int offset = 0;
@@ -370,11 +371,11 @@
 			
 			// test long payload
 			analyzer = new PayloadAnalyzer();
-			writer = new IndexWriter(dir, analyzer, true);
+            writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
 			System.String singleTerm = "lucene";
 			
 			d = new Document();
-			d.Add(new Field(fieldName, singleTerm, Field.Store.NO, Field.Index.TOKENIZED));
+			d.Add(new Field(fieldName, singleTerm, Field.Store.NO, Field.Index.ANALYZED));
 			// add a payload whose length is greater than the buffer size of BufferedIndexOutput
 			payloadData = GenerateRandomData(2000);
 			analyzer.SetPayloadData(fieldName, payloadData, 100, 1500);
@@ -567,10 +568,11 @@
 				this.offset = offset;
 			}
 			
-			public override Token Next(Token token)
+			public override Token Next(Token reusableToken)
 			{
-				token = input.Next(token);
-				if (token != null)
+                System.Diagnostics.Debug.Assert(reusableToken != null);
+				Token nextToken = input.Next(reusableToken);
+				if (nextToken != null)
 				{
 					if (offset + length <= data.Length)
 					{
@@ -578,18 +580,18 @@
 						if (p == null)
 						{
 							p = new Payload();
-							token.SetPayload(p);
+							nextToken.SetPayload(p);
 						}
 						p.SetData(data, offset, length);
 						offset += length;
 					}
 					else
 					{
-						token.SetPayload(null);
+						nextToken.SetPayload(null);
 					}
 				}
 				
-				return token;
+				return nextToken;
 			}
 		}
 		
@@ -601,7 +603,7 @@
 			ByteArrayPool pool = new ByteArrayPool(numThreads, 5);
 			
 			Directory dir = new RAMDirectory();
-			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer());
+            IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
 			System.String field = "test";
 			
 			SupportClass.ThreadClass[] ingesters = new SupportClass.ThreadClass[numThreads];
@@ -633,8 +635,9 @@
 					for (int i = 0; i < freq; i++)
 					{
 						tp.NextPosition();
-						System.String s = System.Text.Encoding.UTF8.GetString(tp.GetPayload(new byte[5], 0));
-						Assert.AreEqual(s, terms.Term().text_ForNUnitTest);
+                        //System.String s = System.Text.Encoding.UTF8.GetString(tp.GetPayload(new byte[5], 0));
+                        //Assert.AreEqual(s, terms.Term().text_ForNUnitTest);
+                        Assert.AreEqual(pool.BytesToString(tp.GetPayload(new byte[5], 0)), terms.Term().text_ForNUnitTest);
 					}
 				}
 				tp.Close();
@@ -650,22 +653,23 @@
 			private byte[] payload;
 			private bool first;
 			private ByteArrayPool pool;
-			
+            private string term;
 			internal PoolingPayloadTokenStream(ByteArrayPool pool)
 			{
 				this.pool = pool;
 				payload = pool.Get();
 				Lucene.Net.Index.TestPayloads.GenerateRandomData(payload);
+                term = pool.BytesToString(payload);
 				first = true;
 			}
 			
-			public override Token Next()
+			public override Token Next(Token reusableToken)
 			{
 				if (!first)
 					return null;
-				Token t = new Token(System.Text.Encoding.UTF8.GetString(payload), 0, 0);
-				t.SetPayload(new Payload(payload));
-				return t;
+				reusableToken.Reinit(term, 0, 0);
+				reusableToken.SetPayload(new Payload(payload));
+				return reusableToken;
 			}
 			
 			public override void  Close()
@@ -686,7 +690,25 @@
 					pool.Add(new byte[size]);
 				}
 			}
-			
+
+            private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
+
+            internal string BytesToString(byte[] bytes)
+            {
+                lock (this)
+                {
+                    string s = System.Text.Encoding.Default.GetString(bytes);
+                    UnicodeUtil.UTF16toUTF8(s, 0, s.Length, utf8Result);
+                    try{
+                        return System.Text.Encoding.UTF8.GetString(utf8Result.result, 0, utf8Result.length);
+                    }
+                    catch (System.Text.DecoderFallbackException)
+                    {
+                        return null;
+                    }
+                }
+            }
+
 			internal virtual byte[] Get()
 			{
 				lock (this)



Mime
View raw message