lucenenet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dougs...@apache.org
Subject svn commit: r798995 [1/35] - in /incubator/lucene.net/trunk/C#/src: Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Analysis/Standard/ Lucene.Net/Document/ Lucene.Net/Index/ Lucene.Net/QueryParser/ Lucene.Net/Search/ Lucene.Net/Search/Function/ Lucene.Net/...
Date Wed, 29 Jul 2009 18:04:24 GMT
Author: dougsale
Date: Wed Jul 29 18:04:12 2009
New Revision: 798995

URL: http://svn.apache.org/viewvc?rev=798995&view=rev
Log:
Initial Lucene 2.4.0 conversion

Added:
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/BufferedDeletes.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteBlockPool.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteSliceReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ByteSliceWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CharBlockPool.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumer.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocConsumerPerThread.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumer.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerField.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumerPerThread.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumers.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldConsumersPerThread.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessor.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerField.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocFieldProcessorPerThread.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverter.cs   (with props)
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocInverterPerThread.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriterThreadState.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxFieldMergeState.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FreqProxTermsWriterPerThread.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommit.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IntBlockPool.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocConsumer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocConsumerPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocConsumerPerThread.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocEndConsumer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocEndConsumerPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/InvertedDocEndConsumerPerThread.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergeDocIDRemapper.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/NormsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/NormsWriterPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/NormsWriterPerThread.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/RawPostingList.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReadOnlyMultiSegmentReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReadOnlySegmentReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ReusableStringReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldWriterPerThread.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/StoredFieldsWriterPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriterPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsTermsWriterPerThread.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHash.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashConsumer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashConsumerPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashConsumerPerThread.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashPerField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermsHashPerThread.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.4.0-VS2005.csproj
      - copied, changed from r797580, incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.3.2-VS2005.csproj
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.4.0-VS2005.sln
      - copied, changed from r797580, incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.3.2-VS2005.sln
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/DocIdSet.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/DocIdSetIterator.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Payload/PayloadSpanUtil.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/PayloadSpans.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/CheckSumIndexInput.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/CheckSumIndexOutput.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/ArrayUtil.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/BitUtil.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Cache/
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Cache/Cache.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Cache/SimpleLRUCache.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Cache/SimpleMapCache.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/CloseableThreadLocal.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/DocIdBitSet.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/OpenBitSet.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/OpenBitSetDISI.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/OpenBitSetIterator.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/SortedVIntList.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/UnicodeUtil.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestByteSlices.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestCloseableThreadLocal.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestCrash.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterExceptions.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestOmitTf.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestTransactions.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/index.19.cfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.19.nocfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.20.cfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.20.nocfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.21.cfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.21.nocfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.22.cfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.22.nocfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.23.cfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Index/index.23.nocfs.zip   (with props)
    incubator/lucene.net/trunk/C#/src/Test/Search/Payloads/PayloadHelper.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestPayloadSpans.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestFilteredSearch.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestSloppyPhraseQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Store/TestDirectory.cs
    incubator/lucene.net/trunk/C#/src/Test/TestMergeSchedulerExternal.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/Cache/
    incubator/lucene.net/trunk/C#/src/Test/Util/Cache/TestSimpleLRUCache.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/TestOpenBitSet.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/TestSortedVIntList.cs
Removed:
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizer.jj
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/TokenMgrError.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.3.2-VS2005.csproj
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net-2.3.2-VS2005.sln
    incubator/lucene.net/trunk/C#/src/Test/Index/index.prelockless.cfs.zip
    incubator/lucene.net/trunk/C#/src/Test/Index/index.prelockless.nocfs.zip
    incubator/lucene.net/trunk/C#/src/Test/Index/index.presharedstores.cfs.zip
    incubator/lucene.net/trunk/C#/src/Test/Index/index.presharedstores.nocfs.zip
    incubator/lucene.net/trunk/C#/src/Test/TestWeakHashTable.cs
Modified:
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Analyzer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CachingTokenFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharArraySet.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharTokenizer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/KeywordTokenizer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LengthFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LowerCaseFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Package.html
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PorterStemFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/SinkTokenizer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardTokenizerImpl.jflex
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/StopFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/TeeTokenFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Token.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/TokenFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/TokenStream.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Tokenizer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/AssemblyInfo.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/AbstractField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/DateTools.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/Document.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/Field.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/FieldSelectorResult.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/Fieldable.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Document/MapFieldSelector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CheckIndex.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/CompoundFileWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ConcurrentMergeScheduler.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DefaultSkipListWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DirectoryIndexReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/DocumentsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldInfo.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldInfos.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldSortedTermVectorMapper.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FieldsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/FilterIndexReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexCommitPoint.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexDeletionPolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileDeleter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileNameFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexFileNames.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexModifier.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/IndexWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/KeepOnlyLastCommitDeletionPolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/LogMergePolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MergePolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiLevelSkipListReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultiSegmentReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/MultipleTermPositions.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/ParallelReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Payload.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfo.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentInfos.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMergeQueue.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentMerger.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermDocs.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermEnum.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermPositions.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SegmentTermVector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SnapshotDeletionPolicy.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/SortedTermVectorMapper.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/Term.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermBuffer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermInfosReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermInfosWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorEntry.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorEntryFreqSortedComparator.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorOffsetInfo.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsReader.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Index/TermVectorsWriter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Lucene.Net.sln
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/CharStream.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/FastCharStream.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/MultiFieldQueryParser.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/ParseException.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/QueryParser.JJ
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/QueryParser.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/QueryParserConstants.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/QueryParserTokenManager.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/Token.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/QueryParser/TokenMgrError.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/BooleanClause.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/BooleanQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/CachingSpanFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/CachingWrapperFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ComplexExplanation.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ConjunctionScorer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ConstantScoreQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ConstantScoreRangeQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/DisjunctionMaxQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/DisjunctionMaxScorer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ExtendedFieldCacheImpl.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldCache.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldCacheImpl.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldDoc.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldDocSortedHitQueue.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FieldSortedHitQueue.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Filter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FilterManager.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FilteredQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ByteFieldSource.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/CustomScoreQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/DocValues.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/FieldCacheSource.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/FieldScoreQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/FloatFieldSource.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/IntFieldSource.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/OrdFieldSource.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ReverseOrdFieldSource.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ShortFieldSource.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ValueSource.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Function/ValueSourceQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/FuzzyQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Hit.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/HitIterator.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/HitQueue.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Hits.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/IndexSearcher.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/MatchAllDocsQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/MultiPhraseQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/MultiTermQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ParallelMultiSearcher.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Payload/BoostingTermQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/PhraseQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/PhraseQueue.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/PrefixFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/PrefixQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Query.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/QueryFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/QueryTermVector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/QueryWrapperFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/RangeFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/RangeQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/RemoteCachingWrapperFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/ScoreDocComparator.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Scorer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Searchable.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Searcher.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SimilarityDelegator.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SloppyPhraseScorer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Sort.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SortField.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SpanFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SpanFilterResult.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/SpanQueryFilter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/NearSpansOrdered.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/NearSpansUnordered.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanFirstQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanNearQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanNotQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanOrQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanScorer.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/SpanTermQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/Spans/TermSpans.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/TermQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/TopDocCollector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/TopFieldDocCollector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/TopFieldDocs.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Search/WildcardQuery.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/SharpZipLibAdapter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/BufferedIndexInput.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/Directory.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/FSDirectory.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/IndexInput.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/IndexOutput.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/Lock.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/LockStressTest.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/MMapDirectory.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMDirectory.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMFile.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMInputStream.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Store/RAMOutputStream.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/SupportClass.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/BitVector.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/Parameter.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/PriorityQueue.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/SmallFloat.cs
    incubator/lucene.net/trunk/C#/src/Lucene.Net/Util/StringHelper.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TeeSinkTokenTest.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestAnalyzers.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestCachingTokenFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestISOLatin1AccentFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestKeywordAnalyzer.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestLengthFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestPerFieldAnalzyerWrapper.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStandardAnalyzer.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopAnalyzer.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Analysis/TestToken.cs
    incubator/lucene.net/trunk/C#/src/Test/AnalysisTest.cs
    incubator/lucene.net/trunk/C#/src/Test/App.config
    incubator/lucene.net/trunk/C#/src/Test/AssemblyInfo.cs
    incubator/lucene.net/trunk/C#/src/Test/Document/TestBinaryDocument.cs
    incubator/lucene.net/trunk/C#/src/Test/Document/TestDocument.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/DocHelper.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/MockIndexInput.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/Store/TestRAMDirectory.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestAddIndexesNoOptimize.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestAtomicUpdate.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestBackwardsCompatibility.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestCheckIndex.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestCompoundFile.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestConcurrentMergeScheduler.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestDeletionPolicy.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestDoc.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestDocumentWriter.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldInfos.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestFieldsReader.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestFilterIndexReader.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexFileDeleter.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexInput.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexModifier.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexReader.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexReaderReopen.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriter.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterDelete.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterLockRelease.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMergePolicy.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMerging.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyBug.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyProxSkipping.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiLevelSkipList.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiSegmentReader.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestNorms.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentMerger.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentReader.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermDocs.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermEnum.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing2.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestTermVectorsReader.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestTermdocPerf.cs
    incubator/lucene.net/trunk/C#/src/Test/Index/TestThreadedOptimize.cs
    incubator/lucene.net/trunk/C#/src/Test/IndexTest.cs
    incubator/lucene.net/trunk/C#/src/Test/QueryParser/TestMultiAnalyzer.cs
    incubator/lucene.net/trunk/C#/src/Test/QueryParser/TestMultiFieldQueryParser.cs
    incubator/lucene.net/trunk/C#/src/Test/QueryParser/TestQueryParser.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/BaseTestRangeFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/CachingWrapperFilterHelper.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/CheckHits.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Function/FunctionTestSetup.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Function/TestCustomScoreQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Function/TestFieldScoreQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Function/TestOrdValues.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/MockFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Payloads/TestBoostingTermQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/QueryUtils.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/RemoteCachingWrapperFilterHelper.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/SingleDocTestFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestBasics.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestNearSpansOrdered.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestSpans.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestSpansAdvanced.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/Spans/TestSpansAdvanced2.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestBoolean2.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestBooleanMinShouldMatch.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestBooleanOr.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestBooleanPrefixQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestBooleanScorer.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestCachingWrapperFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestConstantScoreRangeQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestCustomSearcherSort.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestDateFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestDateSort.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestDisjunctionMaxQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestDocBoost.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestExplanations.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestExtendedFieldCache.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestFilteredQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestFuzzyQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestMatchAllDocsQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestMultiPhraseQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestMultiSearcher.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestMultiSearcherRanking.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestMultiThreadTermVectors.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestNot.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestPhrasePrefixQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestPhraseQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestPositionIncrement.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestPrefixFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestPrefixQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestRangeFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestRangeQuery.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestRemoteCachingWrapperFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestRemoteSearchable.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestScorerPerf.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestSearchHitsWithDeletions.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestSetNorm.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestSimilarity.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestSimpleExplanations.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestSort.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestSpanQueryFilter.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestTermScorer.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestTermVectors.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestThreadSafe.cs
    incubator/lucene.net/trunk/C#/src/Test/Search/TestWildcard.cs
    incubator/lucene.net/trunk/C#/src/Test/SearchTest.cs
    incubator/lucene.net/trunk/C#/src/Test/SearchTestForDuplicates.cs
    incubator/lucene.net/trunk/C#/src/Test/Store/MockRAMDirectory.cs
    incubator/lucene.net/trunk/C#/src/Test/Store/MockRAMInputStream.cs
    incubator/lucene.net/trunk/C#/src/Test/Store/MockRAMOutputStream.cs
    incubator/lucene.net/trunk/C#/src/Test/Store/TestBufferedIndexInput.cs
    incubator/lucene.net/trunk/C#/src/Test/Store/TestHugeRamFile.cs
    incubator/lucene.net/trunk/C#/src/Test/Store/TestLockFactory.cs
    incubator/lucene.net/trunk/C#/src/Test/Store/TestWindowsMMap.cs
    incubator/lucene.net/trunk/C#/src/Test/Test-VS2005.csproj
    incubator/lucene.net/trunk/C#/src/Test/Test.csproj
    incubator/lucene.net/trunk/C#/src/Test/Test.sln
    incubator/lucene.net/trunk/C#/src/Test/TestDemo.cs
    incubator/lucene.net/trunk/C#/src/Test/TestHitIterator.cs
    incubator/lucene.net/trunk/C#/src/Test/TestSearch.cs
    incubator/lucene.net/trunk/C#/src/Test/TestSearchForDuplicates.cs
    incubator/lucene.net/trunk/C#/src/Test/TestSnapshotDeletionPolicy.cs
    incubator/lucene.net/trunk/C#/src/Test/ThreadSafetyTest.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/LuceneTestCase.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/TestSmallFloat.cs
    incubator/lucene.net/trunk/C#/src/Test/Util/_TestUtil.cs

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Analyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/Analyzer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Analyzer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Analyzer.cs Wed Jul 29 18:04:12 2009
@@ -26,17 +26,11 @@
 	/// Typical implementations first build a Tokenizer, which breaks the stream of
 	/// characters from the Reader into raw Tokens.  One or more TokenFilters may
 	/// then be applied to the output of the Tokenizer.
-	/// <p>
-	/// WARNING: You must override one of the methods defined by this class in your
-	/// subclass or the Analyzer will enter an infinite loop.
 	/// </summary>
 	public abstract class Analyzer
 	{
 		/// <summary>Creates a TokenStream which tokenizes all the text in the provided
-		/// Reader.  Default implementation forwards to tokenStream(Reader) for 
-		/// compatibility with older version.  Override to allow Analyzer to choose 
-		/// strategy based on document and/or field.  Must be able to handle null
-		/// field name for backward compatibility. 
+		/// Reader.  Must be able to handle null field name for backward compatibility. 
 		/// </summary>
 		public abstract TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader);
 
@@ -58,7 +52,7 @@
         /// to retrieve previously saved TokenStreams for re-use
         /// by the same thread. 
         /// </summary>
-        protected internal virtual System.Object GetPreviousTokenStream()
+        protected internal virtual object GetPreviousTokenStream()
         {
             return System.Threading.Thread.GetData(tokenStreams);
         }
@@ -67,7 +61,7 @@
         /// to save a TokenStream for later re-use by the same
         /// thread. 
         /// </summary>
-        protected internal virtual void SetPreviousTokenStream(System.Object obj)
+        protected internal virtual void SetPreviousTokenStream(object obj)
         {
             System.Threading.Thread.SetData(tokenStreams, obj);
         }

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CachingTokenFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/CachingTokenFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CachingTokenFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CachingTokenFilter.cs Wed Jul 29 18:04:12 2009
@@ -38,13 +38,14 @@
 		{
 		}
 		
-		public override Token Next()
+		public override Token Next(/* in */ Token reusableToken)
 		{
+            System.Diagnostics.Debug.Assert(reusableToken != null);
 			if (cache == null)
 			{
 				// fill cache lazily
 				cache = new System.Collections.ArrayList();
-				FillCache();
+                FillCache(reusableToken);
 				iterator = cache.GetEnumerator();
 			}
 			
@@ -54,7 +55,8 @@
 				return null;
 			}
 			
-			return (Token) iterator.Current;
+            Token nextToken = (Token) iterator.Current;
+            return (Token) nextToken.Clone();
 		}
 		
 		public override void  Reset()
@@ -65,13 +67,10 @@
 			}
 		}
 		
-		private void  FillCache()
+		private void  FillCache(/* in */ Token reusableToken)
 		{
-			Token token;
-			while ((token = input.Next()) != null)
-			{
-				cache.Add(token);
-			}
+            for (Token nextToken = input.Next(reusableToken); nextToken != null; nextToken = input.Next(reusableToken))
+				cache.Add(nextToken.Clone());
 		}
 	}
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharArraySet.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/CharArraySet.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharArraySet.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharArraySet.cs Wed Jul 29 18:04:12 2009
@@ -194,7 +194,7 @@
 			return count == 0;
 		}
 		
-		public override bool Contains(System.Object o)
+		public override bool Contains(object o)
 		{
 			if (o is char[])
 			{
@@ -209,7 +209,7 @@
 			return false;
 		}
 		
-		public virtual bool Add(System.Object o)
+		public virtual bool Add(object o)
 		{
 			if (o is char[])
 			{
@@ -238,7 +238,7 @@
 			/// <summary>Returns the next String, as a Set<String> would...
 			/// use nextCharArray() for better efficiency. 
 			/// </summary>
-			public virtual System.Object Current
+			public virtual object Current
 			{
 				get
 				{

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharTokenizer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/CharTokenizer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharTokenizer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/CharTokenizer.cs Wed Jul 29 18:04:12 2009
@@ -48,19 +48,20 @@
             return c;
         }
 
-        public override Token Next(Token token)
+        public override Token Next(/* in */ Token reusableToken)
         {
-            token.Clear();
+            System.Diagnostics.Debug.Assert(reusableToken != null);
+            reusableToken.Clear();
             int length = 0;
             int start = bufferIndex;
-            char[] buffer = token.TermBuffer();
+            char[] buffer = reusableToken.TermBuffer();
             while (true)
             {
 
                 if (bufferIndex >= dataLen)
                 {
                     offset += dataLen;
-                    dataLen = input is Lucene.Net.Index.DocumentsWriter.ReusableStringReader ? ((Lucene.Net.Index.DocumentsWriter.ReusableStringReader) input).Read(ioBuffer) : input.Read((System.Char[]) ioBuffer, 0, ioBuffer.Length);
+                    dataLen = input is Lucene.Net.Index.ReusableStringReader ? ((Lucene.Net.Index.ReusableStringReader) input).Read(ioBuffer) : input.Read((System.Char[]) ioBuffer, 0, ioBuffer.Length);
                     if (dataLen <= 0)
                     {
                         if (length > 0)
@@ -81,7 +82,7 @@
                         // start of token
                         start = offset + bufferIndex - 1;
                     else if (length == buffer.Length)
-                        buffer = token.ResizeTermBuffer(1 + length);
+                        buffer = reusableToken.ResizeTermBuffer(1 + length);
 
                     buffer[length++] = Normalize(c); // buffer it, normalized
 
@@ -94,10 +95,10 @@
                     break; // return 'em
             }
 
-            token.termLength = length;
-            token.startOffset = start;
-            token.endOffset = start + length;
-            return token;
+            reusableToken.SetTermLength(length);
+            reusableToken.SetStartOffset(start);
+            reusableToken.SetEndOffset(start + length);
+            return reusableToken;
         }
 
         public override void Reset(System.IO.TextReader input)

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/ISOLatin1AccentFilter.cs Wed Jul 29 18:04:12 2009
@@ -35,26 +35,27 @@
         private char[] output = new char[256];
         private int outputPos;
 
-        public override Token Next(Token result)
+        public override Token Next(/* in */ Token reusableToken)
 		{
-            result = input.Next(result);
-            if (result != null)
+            System.Diagnostics.Debug.Assert(reusableToken != null);
+            Token nextToken = input.Next(reusableToken);
+            if (nextToken != null)
             {
-                char[] buffer = result.TermBuffer();
-                int length = result.TermLength();
+                char[] buffer = nextToken.TermBuffer();
+                int length = nextToken.TermLength();
                 // If no characters actually require rewriting then we
                 // just return token as-is:
                 for (int i = 0; i < length; i++)
                 {
                     char c = buffer[i];
-                    if (c >= '\u00c0' && c <= '\u0178')
+                    if (c >= '\u00c0' && c <= '\ufb06')
                     {
                         RemoveAccents(buffer, length);
-                        result.SetTermBuffer(output, 0, outputPos);
+                        nextToken.SetTermBuffer(output, 0, outputPos);
                         break;
                     }
                 }
-                return result;
+                return nextToken;
             }
             else
                 return null;
@@ -84,7 +85,7 @@
 
                 // Quick test: if it's not in range then just keep
                 // current character
-                if (c < '\u00c0')
+                if (c < '\u00c0' || c > '\ufb06')
                     output[outputPos++] = c;
                 else
                 {
@@ -134,6 +135,11 @@
                             output[outputPos++] = 'I';
                             break;
 
+                        case '\u0132': // IJ
+                            output[outputPos++] = 'I';
+                            output[outputPos++] = 'J';
+                            break;
+
                         case '\u00D0':  // Ð
                             output[outputPos++] = 'D';
                             break;
@@ -225,6 +231,11 @@
                             output[outputPos++] = 'i';
                             break;
 
+                        case '\u0133': // ij
+                            output[outputPos++] = 'i';
+                            output[outputPos++] = 'j';
+                            break;
+
                         case '\u00F0':  // ð
                             output[outputPos++] = 'd';
                             break;
@@ -278,6 +289,38 @@
                             output[outputPos++] = 'y';
                             break;
 
+                        case '\uFB00': // ff
+                            output[outputPos++] = 'f';
+                            output[outputPos++] = 'f';
+                            break;
+                        case '\uFB01': // fi
+                            output[outputPos++] = 'f';
+                            output[outputPos++] = 'i';
+                            break;
+                        case '\uFB02': // fl
+                            output[outputPos++] = 'f';
+                            output[outputPos++] = 'l';
+                            break;
+                        // following 2 are commented as they can break the maxSizeNeeded (and doing *3 could be expensive)
+                        //        case '\uFB03': // ffi
+                        //            output[outputPos++] = 'f';
+                        //            output[outputPos++] = 'f';
+                        //            output[outputPos++] = 'i';
+                        //            break;
+                        //        case '\uFB04': // ffl
+                        //            output[outputPos++] = 'f';
+                        //            output[outputPos++] = 'f';
+                        //            output[outputPos++] = 'l';
+                        //            break;
+                        case '\uFB05': // ſt
+                            output[outputPos++] = 'f';
+                            output[outputPos++] = 't';
+                            break;
+                        case '\uFB06': // st
+                            output[outputPos++] = 's';
+                            output[outputPos++] = 't';
+                            break;
+
                         default:
                             output[outputPos++] = c;
                             break;

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/KeywordTokenizer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/KeywordTokenizer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/KeywordTokenizer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/KeywordTokenizer.cs Wed Jul 29 18:04:12 2009
@@ -37,14 +37,15 @@
             this.done = false;
         }
 
-        public override Token Next(Token result)
+        public override Token Next(/* in */ Token reusableToken)
         {
+            System.Diagnostics.Debug.Assert(reusableToken != null);
             if (!done)
             {
                 done = true;
                 int upto = 0;
-                result.Clear();
-                char[] buffer = result.TermBuffer();
+                reusableToken.Clear();
+                char[] buffer = reusableToken.TermBuffer();
                 while (true)
                 {
                     int length = input.Read(buffer, upto, buffer.Length - upto);
@@ -52,10 +53,10 @@
                         break;
                     upto += length;
                     if (upto == buffer.Length)
-                        buffer = result.ResizeTermBuffer(1 + buffer.Length);
+                        buffer = reusableToken.ResizeTermBuffer(1 + buffer.Length);
                 }
-                result.termLength = upto;
-                return result;
+                reusableToken.SetTermLength(upto);
+                return reusableToken;
             }
             return null;
         }

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LengthFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/LengthFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LengthFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LengthFilter.cs Wed Jul 29 18:04:12 2009
@@ -41,16 +41,17 @@
             this.max = max;
         }
 		
-        /// <summary> Returns the next input Token whose termText() is the right len</summary>
-        public override Token Next(Token result)
+        /// <summary> Returns the next input Token whose term() is the right len</summary>
+        public override Token Next(/* in */ Token reusableToken)
         {
+            System.Diagnostics.Debug.Assert(reusableToken != null);
             // return the first non-stop word found
-            for (Token token = input.Next(result); token != null; token = input.Next(result))
+            for (Token nextToken = input.Next(reusableToken); nextToken != null; nextToken = input.Next(reusableToken))
             {
-                int len = token.TermText().Length;
+                int len = nextToken.TermLength();
                 if (len >= min && len <= max)
                 {
-                    return token;
+                    return nextToken;
                 }
                 // note: else we ignore it but should we index each part of it?
             }

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LowerCaseFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/LowerCaseFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LowerCaseFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/LowerCaseFilter.cs Wed Jul 29 18:04:12 2009
@@ -31,18 +31,18 @@
         {
         }
 		
-        public override Token Next(Token result)
+        public override Token Next(Token reusableToken)
         {
-            result = input.Next(result);
-            if (result != null)
+            Token nextToken = input.Next(reusableToken);
+            if (nextToken != null)
             {
 
-                char[] buffer = result.TermBuffer();
-                int length = result.termLength;
+                char[] buffer = nextToken.TermBuffer();
+                int length = nextToken.TermLength();
                 for (int i = 0; i < length; i++)
                     buffer[i] = System.Char.ToLower(buffer[i]);
 
-                return result;
+                return nextToken;
             }
             else
                 return null;

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Package.html
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/Package.html?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Package.html (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Package.html Wed Jul 29 18:04:12 2009
@@ -1,256 +1,272 @@
-<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
-<html>
-<head>
-   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
-   <meta name="Author" content="Doug Cutting">
-</head>
-<body>
-<p>API and code to convert text into indexable/searchable tokens.  Covers {@link org.apache.lucene.analysis.Analyzer} and related classes.</p>
-<h2>Parsing? Tokenization? Analysis!</h2>
-<p>
-Lucene, indexing and search library, accepts only plain text input.
-<p>
-<h2>Parsing</h2>
-<p>
-Applications that build their search capabilities upon Lucene may support documents in various formats &ndash; HTML, XML, PDF, Word &ndash; just to name a few.
-Lucene does not care about the <i>Parsing</i> of these and other document formats, and it is the responsibility of the 
-application using Lucene to use an appropriate <i>Parser</i> to convert the original format into plain text before passing that plain text to Lucene.
-<p>
-<h2>Tokenization</h2>
-<p>
-Plain text passed to Lucene for indexing goes through a process generally called tokenization &ndash; namely breaking of the 
-input text into small indexing elements &ndash; 
-{@link org.apache.lucene.analysis.Token Tokens}.
-The way input text is broken into tokens very 
-much dictates further capabilities of search upon that text. 
-For instance, sentences beginnings and endings can be identified to provide for more accurate phrase 
-and proximity searches (though sentence identification is not provided by Lucene).
-<p>
-In some cases simply breaking the input text into tokens is not enough &ndash; a deeper <i>Analysis</i> is needed,
-providing for several functions, including (but not limited to):
-<ul>
-  <li><a href = "http://en.wikipedia.org//wiki/Stemming">Stemming</a> &ndash; 
-      Replacing of words by their stems. 
-      For instance with English stemming "bikes" is replaced by "bike"; 
-      now query "bike" can find both documents containing "bike" and those containing "bikes".
-  </li>
-  <li><a href = "http://en.wikipedia.org//wiki/Stop_words">Stop Words Filtering</a> &ndash; 
-      Common words like "the", "and" and "a" rarely add any value to a search.
-      Removing them shrinks the index size and increases performance.
-      It may also reduce some "noise" and actually improve search quality.
-  </li>
-  <li><a href = "http://en.wikipedia.org//wiki/Text_normalization">Text Normalization</a> &ndash; 
-      Stripping accents and other character markings can make for better searching.
-  </li>
-  <li><a href = "http://en.wikipedia.org//wiki/Synonym">Synonym Expansion</a> &ndash; 
-      Adding in synonyms at the same token position as the current word can mean better 
-      matching when users search with words in the synonym set.
-  </li>
-</ul> 
-<p>
-<h2>Core Analysis</h2>
-<p>
-  The analysis package provides the mechanism to convert Strings and Readers into tokens that can be indexed by Lucene.  There
-  are three main classes in the package from which all analysis processes are derived.  These are:
-  <ul>
-    <li>{@link org.apache.lucene.analysis.Analyzer} &ndash; An Analyzer is responsible for building a {@link org.apache.lucene.analysis.TokenStream} which can be consumed
-    by the indexing and searching processes.  See below for more information on implementing your own Analyzer.</li>
-    <li>{@link org.apache.lucene.analysis.Tokenizer} &ndash; A Tokenizer is a {@link org.apache.lucene.analysis.TokenStream} and is responsible for breaking
-    up incoming text into {@link org.apache.lucene.analysis.Token}s.  In most cases, an Analyzer will use a Tokenizer as the first step in
-    the analysis process.</li>
-    <li>{@link org.apache.lucene.analysis.TokenFilter} &ndash; A TokenFilter is also a {@link org.apache.lucene.analysis.TokenStream} and is responsible
-    for modifying {@link org.apache.lucene.analysis.Token}s that have been created by the Tokenizer.  Common modifications performed by a
-    TokenFilter are: deletion, stemming, synonym injection, and down casing.  Not all Analyzers require TokenFilters</li>
-  </ul>
-</p>
-<h2>Hints, Tips and Traps</h2>
-<p>
-   The synergy between {@link org.apache.lucene.analysis.Analyzer} and {@link org.apache.lucene.analysis.Tokenizer}
-   is sometimes confusing. To ease on this confusion, some clarifications:
-   <ul>
-      <li>The {@link org.apache.lucene.analysis.Analyzer} is responsible for the entire task of 
-          <u>creating</u> tokens out of the input text, while the {@link org.apache.lucene.analysis.Tokenizer}
-          is only responsible for <u>breaking</u> the input text into tokens. Very likely, tokens created 
-          by the {@link org.apache.lucene.analysis.Tokenizer} would be modified or even omitted 
-          by the {@link org.apache.lucene.analysis.Analyzer} (via one or more
-          {@link org.apache.lucene.analysis.TokenFilter}s) before being returned.
-       </li>
-       <li>{@link org.apache.lucene.analysis.Tokenizer} is a {@link org.apache.lucene.analysis.TokenStream}, 
-           but {@link org.apache.lucene.analysis.Analyzer} is not.
-       </li>
-       <li>{@link org.apache.lucene.analysis.Analyzer} is "field aware", but 
-           {@link org.apache.lucene.analysis.Tokenizer} is not.
-       </li>
-   </ul>
-</p>
-<p>
-  Lucene Java provides a number of analysis capabilities, the most commonly used one being the {@link
-  org.apache.lucene.analysis.standard.StandardAnalyzer}.  Many applications will have a long and industrious life with nothing more
-  than the StandardAnalyzer.  However, there are a few other classes/packages that are worth mentioning:
-  <ol>
-    <li>{@link org.apache.lucene.analysis.PerFieldAnalyzerWrapper} &ndash; Most Analyzers perform the same operation on all
-      {@link org.apache.lucene.document.Field}s.  The PerFieldAnalyzerWrapper can be used to associate a different Analyzer with different
-      {@link org.apache.lucene.document.Field}s.</li>
-    <li>The contrib/analyzers library located at the root of the Lucene distribution has a number of different Analyzer implementations to solve a variety
-    of different problems related to searching.  Many of the Analyzers are designed to analyze non-English languages.</li>
-    <li>The {@link org.apache.lucene.analysis.snowball contrib/snowball library} 
-        located at the root of the Lucene distribution has Analyzer and TokenFilter 
-        implementations for a variety of Snowball stemmers.  
-        See <a href = "http://snowball.tartarus.org">http://snowball.tartarus.org</a> 
-        for more information on Snowball stemmers.</li>
-    <li>There are a variety of Tokenizer and TokenFilter implementations in this package.  Take a look around, chances are someone has implemented what you need.</li>
-  </ol>
-</p>
-<p>
-  Analysis is one of the main causes of performance degradation during indexing.  Simply put, the more you analyze the slower the indexing (in most cases).
-  Perhaps your application would be just fine using the simple {@link org.apache.lucene.analysis.WhitespaceTokenizer} combined with a
-  {@link org.apache.lucene.analysis.StopFilter}. The contrib/benchmark library can be useful for testing out the speed of the analysis process.
-</p>
-<h2>Invoking the Analyzer</h2>
-<p>
-  Applications usually do not invoke analysis &ndash; Lucene does it for them:
-  <ul>
-    <li>At indexing, as a consequence of 
-        {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document) addDocument(doc)},
-        the Analyzer in effect for indexing is invoked for each indexed field of the added document.
-    </li>
-    <li>At search, as a consequence of
-        {@link org.apache.lucene.queryParser.QueryParser#parse(java.lang.String) QueryParser.parse(queryText)},
-        the QueryParser may invoke the Analyzer in effect.
-        Note that for some queries analysis does not take place, e.g. wildcard queries.
-    </li>
-  </ul>
-  However an application might invoke Analysis of any text for testing or for any other purpose, something like:
-  <PRE>
-      Analyzer analyzer = new StandardAnalyzer(); // or any other analyzer
-      TokenStream ts = analyzer.tokenStream("myfield",new StringReader("some text goes here"));
-      Token t = ts.next();
-      while (t!=null) {
-        System.out.println("token: "+t));
-        t = ts.next();
-      }
-  </PRE>
-</p>
-<h2>Indexing Analysis vs. Search Analysis</h2>
-<p>
-  Selecting the "correct" analyzer is crucial
-  for search quality, and can also affect indexing and search performance.
-  The "correct" analyzer differs between applications.
-  Lucene java's wiki page 
-  <a href = "http://wiki.apache.org//lucene-java/AnalysisParalysis">AnalysisParalysis</a> 
-  provides some data on "analyzing your analyzer".
-  Here are some rules of thumb:
-  <ol>
-    <li>Test test test... (did we say test?)</li>
-    <li>Beware of over analysis &ndash; might hurt indexing performance.</li>
-    <li>Start with same analyzer for indexing and search, otherwise searches would not find what they are supposed to...</li>
-    <li>In some cases a different analyzer is required for indexing and search, for instance:
-        <ul>
-           <li>Certain searches require more stop words to be filtered. (I.e. more than those that were filtered at indexing.)</li>
-           <li>Query expansion by synonyms, acronyms, auto spell correction, etc.</li>
-        </ul>
-        This might sometimes require a modified analyzer &ndash; see the next section on how to do that.
-    </li>
-  </ol>
-</p>
-<h2>Implementing your own Analyzer</h2>
-<p>Creating your own Analyzer is straightforward. It usually involves either wrapping an existing Tokenizer and  set of TokenFilters to create a new Analyzer
-or creating both the Analyzer and a Tokenizer or TokenFilter.  Before pursuing this approach, you may find it worthwhile
-to explore the contrib/analyzers library and/or ask on the java-user@lucene.apache.org mailing list first to see if what you need already exists.
-If you are still committed to creating your own Analyzer or TokenStream derivation (Tokenizer or TokenFilter) have a look at
-the source code of any one of the many samples located in this package.
-</p>
-<p>
-  The following sections discuss some aspects of implementing your own analyzer.
-</p>
-<h3>Field Section Boundaries</h2>
-<p>
-  When {@link org.apache.lucene.document.Document#add(org.apache.lucene.document.Fieldable) document.add(field)}
-  is called multiple times for the same field name, we could say that each such call creates a new 
-  section for that field in that document. 
-  In fact, a separate call to 
-  {@link org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String, java.io.Reader) tokenStream(field,reader)}
-  would take place for each of these so called "sections".
-  However, the default Analyzer behavior is to treat all these sections as one large section. 
-  This allows phrase search and proximity search to seamlessly cross 
-  boundaries between these "sections".
-  In other words, if a certain field "f" is added like this:
-  <PRE>
-      document.add(new Field("f","first ends",...);
-      document.add(new Field("f","starts two",...);
-      indexWriter.addDocument(document);
-  </PRE>
-  Then, a phrase search for "ends starts" would find that document.
-  Where desired, this behavior can be modified by introducing a "position gap" between consecutive field "sections", 
-  simply by overriding 
-  {@link org.apache.lucene.analysis.Analyzer#getPositionIncrementGap(java.lang.String) Analyzer.getPositionIncrementGap(fieldName)}:
-  <PRE>
-      Analyzer myAnalyzer = new StandardAnalyzer() {
-         public int getPositionIncrementGap(String fieldName) {
-           return 10;
-         }
-      };
-  </PRE>
-</p>
-<h3>Token Position Increments</h2>
-<p>
-   By default, all tokens created by Analyzers and Tokenizers have a 
-   {@link org.apache.lucene.analysis.Token#getPositionIncrement() position increment} of one.
-   This means that the position stored for that token in the index would be one more than
-   that of the previous token.
-   Recall that phrase and proximity searches rely on position info.
-</p>
-<p>
-   If the selected analyzer filters the stop words "is" and "the", then for a document 
-   containing the string "blue is the sky", only the tokens "blue", "sky" are indexed, 
-   with position("sky") = 1 + position("blue"). Now, a phrase query "blue is the sky"
-   would find that document, because the same analyzer filters the same stop words from
-   that query. But also the phrase query "blue sky" would find that document.
-</p>
-<p>   
-   If this behavior does not fit the application needs,
-   a modified analyzer can be used, that would increment further the positions of
-   tokens following a removed stop word, using
-   {@link org.apache.lucene.analysis.Token#setPositionIncrement(int)}.
-   This can be done with something like:
-   <PRE>
-      public TokenStream tokenStream(final String fieldName, Reader reader) {
-        final TokenStream ts = someAnalyzer.tokenStream(fieldName, reader);
-        TokenStream res = new TokenStream() {
-          public Token next() throws IOException {
-            int extraIncrement = 0;
-            while (true) {
-              Token t = ts.next();
-              if (t!=null) {
-                if (stopWords.contains(t.termText())) {
-                  extraIncrement++; // filter this word
-                  continue;
-                } 
-                if (extraIncrement>0) {
-                  t.setPositionIncrement(t.getPositionIncrement()+extraIncrement);
-                }
-              }
-              return t;
-            }
-          }
-        };
-        return res;
-      }
-   </PRE>
-   Now, with this modified analyzer, the phrase query "blue sky" would find that document.
-   But note that this is yet not a perfect solution, because any phrase query "blue w1 w2 sky"
-   where both w1 and w2 are stop words would match that document.
-</p>
-<p>
-   Few more use cases for modifying position increments are:
-   <ol>
-     <li>Inhibiting phrase and proximity matches in sentence boundaries &ndash; for this, a tokenizer that 
-         identifies a new sentence can add 1 to the position increment of the first token of the new sentence.</li>
-     <li>Injecting synonyms &ndash; here, synonyms of a token should be added after that token, 
-         and their position increment should be set to 0.
-         As result, all synonyms of a token would be considered to appear in exactly the 
-         same position as that token, and so would they be seen by phrase and proximity searches.</li>
-   </ol>
-</p>
-</body>
-</html>
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+   <meta name="Author" content="Doug Cutting">
+</head>
+<body>
+<p>API and code to convert text into indexable/searchable tokens.  Covers {@link org.apache.lucene.analysis.Analyzer} and related classes.</p>
+<h2>Parsing? Tokenization? Analysis!</h2>
+<p>
+Lucene, indexing and search library, accepts only plain text input.
+<p>
+<h2>Parsing</h2>
+<p>
+Applications that build their search capabilities upon Lucene may support documents in various formats &ndash; HTML, XML, PDF, Word &ndash; just to name a few.
+Lucene does not care about the <i>Parsing</i> of these and other document formats, and it is the responsibility of the 
+application using Lucene to use an appropriate <i>Parser</i> to convert the original format into plain text before passing that plain text to Lucene.
+<p>
+<h2>Tokenization</h2>
+<p>
+Plain text passed to Lucene for indexing goes through a process generally called tokenization &ndash; namely breaking of the 
+input text into small indexing elements &ndash; 
+{@link org.apache.lucene.analysis.Token Tokens}.
+The way input text is broken into tokens very 
+much dictates further capabilities of search upon that text. 
+For instance, sentences beginnings and endings can be identified to provide for more accurate phrase 
+and proximity searches (though sentence identification is not provided by Lucene).
+<p>
+In some cases simply breaking the input text into tokens is not enough &ndash; a deeper <i>Analysis</i> is needed,
+providing for several functions, including (but not limited to):
+<ul>
+  <li><a href="http://en.wikipedia.org/wiki/Stemming">Stemming</a> &ndash; 
+      Replacing of words by their stems. 
+      For instance with English stemming "bikes" is replaced by "bike"; 
+      now query "bike" can find both documents containing "bike" and those containing "bikes".
+  </li>
+  <li><a href="http://en.wikipedia.org/wiki/Stop_words">Stop Words Filtering</a> &ndash; 
+      Common words like "the", "and" and "a" rarely add any value to a search.
+      Removing them shrinks the index size and increases performance.
+      It may also reduce some "noise" and actually improve search quality.
+  </li>
+  <li><a href="http://en.wikipedia.org/wiki/Text_normalization">Text Normalization</a> &ndash; 
+      Stripping accents and other character markings can make for better searching.
+  </li>
+  <li><a href="http://en.wikipedia.org/wiki/Synonym">Synonym Expansion</a> &ndash; 
+      Adding in synonyms at the same token position as the current word can mean better 
+      matching when users search with words in the synonym set.
+  </li>
+</ul> 
+<p>
+<h2>Core Analysis</h2>
+<p>
+  The analysis package provides the mechanism to convert Strings and Readers into tokens that can be indexed by Lucene.  There
+  are three main classes in the package from which all analysis processes are derived.  These are:
+  <ul>
+    <li>{@link org.apache.lucene.analysis.Analyzer} &ndash; An Analyzer is responsible for building a {@link org.apache.lucene.analysis.TokenStream} which can be consumed
+    by the indexing and searching processes.  See below for more information on implementing your own Analyzer.</li>
+    <li>{@link org.apache.lucene.analysis.Tokenizer} &ndash; A Tokenizer is a {@link org.apache.lucene.analysis.TokenStream} and is responsible for breaking
+    up incoming text into {@link org.apache.lucene.analysis.Token}s.  In most cases, an Analyzer will use a Tokenizer as the first step in
+    the analysis process.</li>
+    <li>{@link org.apache.lucene.analysis.TokenFilter} &ndash; A TokenFilter is also a {@link org.apache.lucene.analysis.TokenStream} and is responsible
+    for modifying {@link org.apache.lucene.analysis.Token}s that have been created by the Tokenizer.  Common modifications performed by a
+    TokenFilter are: deletion, stemming, synonym injection, and down casing.  Not all Analyzers require TokenFilters</li>
+  </ul>
+</p>
+<h2>Hints, Tips and Traps</h2>
+<p>
+   The synergy between {@link org.apache.lucene.analysis.Analyzer} and {@link org.apache.lucene.analysis.Tokenizer}
+   is sometimes confusing. To ease on this confusion, some clarifications:
+   <ul>
+      <li>The {@link org.apache.lucene.analysis.Analyzer} is responsible for the entire task of 
+          <u>creating</u> tokens out of the input text, while the {@link org.apache.lucene.analysis.Tokenizer}
+          is only responsible for <u>breaking</u> the input text into tokens. Very likely, tokens created 
+          by the {@link org.apache.lucene.analysis.Tokenizer} would be modified or even omitted 
+          by the {@link org.apache.lucene.analysis.Analyzer} (via one or more
+          {@link org.apache.lucene.analysis.TokenFilter}s) before being returned.
+       </li>
+       <li>{@link org.apache.lucene.analysis.Tokenizer} is a {@link org.apache.lucene.analysis.TokenStream}, 
+           but {@link org.apache.lucene.analysis.Analyzer} is not.
+       </li>
+       <li>{@link org.apache.lucene.analysis.Analyzer} is "field aware", but 
+           {@link org.apache.lucene.analysis.Tokenizer} is not.
+       </li>
+   </ul>
+</p>
+<p>
+  Lucene Java provides a number of analysis capabilities, the most commonly used one being the {@link
+  org.apache.lucene.analysis.standard.StandardAnalyzer}.  Many applications will have a long and industrious life with nothing more
+  than the StandardAnalyzer.  However, there are a few other classes/packages that are worth mentioning:
+  <ol>
+    <li>{@link org.apache.lucene.analysis.PerFieldAnalyzerWrapper} &ndash; Most Analyzers perform the same operation on all
+      {@link org.apache.lucene.document.Field}s.  The PerFieldAnalyzerWrapper can be used to associate a different Analyzer with different
+      {@link org.apache.lucene.document.Field}s.</li>
+    <li>The contrib/analyzers library located at the root of the Lucene distribution has a number of different Analyzer implementations to solve a variety
+    of different problems related to searching.  Many of the Analyzers are designed to analyze non-English languages.</li>
+    <li>The contrib/snowball library 
+        located at the root of the Lucene distribution has Analyzer and TokenFilter 
+        implementations for a variety of Snowball stemmers.  
+        See <a href="http://snowball.tartarus.org">http://snowball.tartarus.org</a> 
+        for more information on Snowball stemmers.</li>
+    <li>There are a variety of Tokenizer and TokenFilter implementations in this package.  Take a look around, chances are someone has implemented what you need.</li>
+  </ol>
+</p>
+<p>
+  Analysis is one of the main causes of performance degradation during indexing.  Simply put, the more you analyze the slower the indexing (in most cases).
+  Perhaps your application would be just fine using the simple {@link org.apache.lucene.analysis.WhitespaceTokenizer} combined with a
+  {@link org.apache.lucene.analysis.StopFilter}. The contrib/benchmark library can be useful for testing out the speed of the analysis process.
+</p>
+<h2>Invoking the Analyzer</h2>
+<p>
+  Applications usually do not invoke analysis &ndash; Lucene does it for them:
+  <ul>
+    <li>At indexing, as a consequence of 
+        {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document) addDocument(doc)},
+        the Analyzer in effect for indexing is invoked for each indexed field of the added document.
+    </li>
+    <li>At search, as a consequence of
+        {@link org.apache.lucene.queryParser.QueryParser#parse(java.lang.String) QueryParser.parse(queryText)},
+        the QueryParser may invoke the Analyzer in effect.
+        Note that for some queries analysis does not take place, e.g. wildcard queries.
+    </li>
+  </ul>
+  However an application might invoke Analysis of any text for testing or for any other purpose, something like:
+  <PRE>
+      Analyzer analyzer = new StandardAnalyzer(); // or any other analyzer
+      TokenStream ts = analyzer.tokenStream("myfield",new StringReader("some text goes here"));
+      Token t = ts.next();
+      while (t!=null) {
+        System.out.println("token: "+t));
+        t = ts.next();
+      }
+  </PRE>
+</p>
+<h2>Indexing Analysis vs. Search Analysis</h2>
+<p>
+  Selecting the "correct" analyzer is crucial
+  for search quality, and can also affect indexing and search performance.
+  The "correct" analyzer differs between applications.
+  Lucene java's wiki page 
+  <a href="http://wiki.apache.org/lucene-java/AnalysisParalysis">AnalysisParalysis</a> 
+  provides some data on "analyzing your analyzer".
+  Here are some rules of thumb:
+  <ol>
+    <li>Test test test... (did we say test?)</li>
+    <li>Beware of over analysis &ndash; might hurt indexing performance.</li>
+    <li>Start with same analyzer for indexing and search, otherwise searches would not find what they are supposed to...</li>
+    <li>In some cases a different analyzer is required for indexing and search, for instance:
+        <ul>
+           <li>Certain searches require more stop words to be filtered. (I.e. more than those that were filtered at indexing.)</li>
+           <li>Query expansion by synonyms, acronyms, auto spell correction, etc.</li>
+        </ul>
+        This might sometimes require a modified analyzer &ndash; see the next section on how to do that.
+    </li>
+  </ol>
+</p>
+<h2>Implementing your own Analyzer</h2>
+<p>Creating your own Analyzer is straightforward. It usually involves either wrapping an existing Tokenizer and  set of TokenFilters to create a new Analyzer
+or creating both the Analyzer and a Tokenizer or TokenFilter.  Before pursuing this approach, you may find it worthwhile
+to explore the contrib/analyzers library and/or ask on the java-user@lucene.apache.org mailing list first to see if what you need already exists.
+If you are still committed to creating your own Analyzer or TokenStream derivation (Tokenizer or TokenFilter) have a look at
+the source code of any one of the many samples located in this package.
+</p>
+<p>
+  The following sections discuss some aspects of implementing your own analyzer.
+</p>
+<h3>Field Section Boundaries</h2>
+<p>
+  When {@link org.apache.lucene.document.Document#add(org.apache.lucene.document.Fieldable) document.add(field)}
+  is called multiple times for the same field name, we could say that each such call creates a new 
+  section for that field in that document. 
+  In fact, a separate call to 
+  {@link org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String, java.io.Reader) tokenStream(field,reader)}
+  would take place for each of these so called "sections".
+  However, the default Analyzer behavior is to treat all these sections as one large section. 
+  This allows phrase search and proximity search to seamlessly cross 
+  boundaries between these "sections".
+  In other words, if a certain field "f" is added like this:
+  <PRE>
+      document.add(new Field("f","first ends",...);
+      document.add(new Field("f","starts two",...);
+      indexWriter.addDocument(document);
+  </PRE>
+  Then, a phrase search for "ends starts" would find that document.
+  Where desired, this behavior can be modified by introducing a "position gap" between consecutive field "sections", 
+  simply by overriding 
+  {@link org.apache.lucene.analysis.Analyzer#getPositionIncrementGap(java.lang.String) Analyzer.getPositionIncrementGap(fieldName)}:
+  <PRE>
+      Analyzer myAnalyzer = new StandardAnalyzer() {
+         public int getPositionIncrementGap(String fieldName) {
+           return 10;
+         }
+      };
+  </PRE>
+</p>
+<h3>Token Position Increments</h2>
+<p>
+   By default, all tokens created by Analyzers and Tokenizers have a 
+   {@link org.apache.lucene.analysis.Token#getPositionIncrement() position increment} of one.
+   This means that the position stored for that token in the index would be one more than
+   that of the previous token.
+   Recall that phrase and proximity searches rely on position info.
+</p>
+<p>
+   If the selected analyzer filters the stop words "is" and "the", then for a document 
+   containing the string "blue is the sky", only the tokens "blue", "sky" are indexed, 
+   with position("sky") = 1 + position("blue"). Now, a phrase query "blue is the sky"
+   would find that document, because the same analyzer filters the same stop words from
+   that query. But also the phrase query "blue sky" would find that document.
+</p>
+<p>   
+   If this behavior does not fit the application needs,
+   a modified analyzer can be used, that would increment further the positions of
+   tokens following a removed stop word, using
+   {@link org.apache.lucene.analysis.Token#setPositionIncrement(int)}.
+   This can be done with something like:
+   <PRE>
+      public TokenStream tokenStream(final String fieldName, Reader reader) {
+        final TokenStream ts = someAnalyzer.tokenStream(fieldName, reader);
+        TokenStream res = new TokenStream() {
+          public Token next() throws IOException {
+            int extraIncrement = 0;
+            while (true) {
+              Token t = ts.next();
+              if (t!=null) {
+                if (stopWords.contains(t.termText())) {
+                  extraIncrement++; // filter this word
+                  continue;
+                } 
+                if (extraIncrement>0) {
+                  t.setPositionIncrement(t.getPositionIncrement()+extraIncrement);
+                }
+              }
+              return t;
+            }
+          }
+        };
+        return res;
+      }
+   </PRE>
+   Now, with this modified analyzer, the phrase query "blue sky" would find that document.
+   But note that this is yet not a perfect solution, because any phrase query "blue w1 w2 sky"
+   where both w1 and w2 are stop words would match that document.
+</p>
+<p>
+   Few more use cases for modifying position increments are:
+   <ol>
+     <li>Inhibiting phrase and proximity matches in sentence boundaries &ndash; for this, a tokenizer that 
+         identifies a new sentence can add 1 to the position increment of the first token of the new sentence.</li>
+     <li>Injecting synonyms &ndash; here, synonyms of a token should be added after that token, 
+         and their position increment should be set to 0.
+         As result, all synonyms of a token would be considered to appear in exactly the 
+         same position as that token, and so would they be seen by phrase and proximity searches.</li>
+   </ol>
+</p>
+</body>
+</html>

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PorterStemFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/PorterStemFilter.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PorterStemFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/PorterStemFilter.cs Wed Jul 29 18:04:12 2009
@@ -47,17 +47,17 @@
             stemmer = new PorterStemmer();
         }
 
-        public override Token Next(Token result)
+        public override Token Next(/* in */ Token reusableToken)
         {
-            result = input.Next(result);
-            if (result != null)
-            {
-                if (stemmer.Stem(result.TermBuffer(), 0, result.termLength))
-                    result.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength());
-                return result;
-            }
-            else
+            System.Diagnostics.Debug.Assert(reusableToken != null);
+            Token nextToken = input.Next(reusableToken);
+            if (nextToken == null)
                 return null;
+
+            if (stemmer.Stem(nextToken.TermBuffer(), 0, nextToken.TermLength()))
+                nextToken.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength());
+
+            return nextToken;
         }
     }
 }
\ No newline at end of file

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/SinkTokenizer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/SinkTokenizer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/SinkTokenizer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/SinkTokenizer.cs Wed Jul 29 18:04:12 2009
@@ -55,7 +55,8 @@
 		/// WARNING: Adding tokens to this list requires the {@link #Reset()} method to be called in order for them
 		/// to be made available.  Also, this Tokenizer does nothing to protect against {@link java.util.ConcurrentModificationException}s
 		/// in the case of adds happening while {@link #Next(Lucene.Net.Analysis.Token)} is being called.
-		/// 
+		/// <p/>
+        /// WARNING: Since this SinkTokenizer can be reset and the cached tokens made available again, do not modify them.  Modify clones instead.
 		/// </summary>
 		/// <returns> A List of {@link Lucene.Net.Analysis.Token}s
 		/// </returns>
@@ -68,11 +69,17 @@
 		/// <returns> The next {@link Lucene.Net.Analysis.Token} in the Sink.
 		/// </returns>
 		/// <throws>  IOException </throws>
-		public override Token Next()
+		public override Token Next(/* in */ Token reusableToken)
 		{
+            System.Diagnostics.Debug.Assert(reusableToken != null); 
 			if (iter == null)
 				iter = lst.GetEnumerator();
-			return iter.MoveNext() ? (Token) iter.Current : null;
+            if (iter.MoveNext())
+            {
+                Token nextToken = (Token) iter.Current;
+                return (Token) nextToken.Clone();
+            }
+            return null;
 		}
 		
 		

Modified: incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Lucene.Net/Analysis/Standard/StandardAnalyzer.cs Wed Jul 29 18:04:12 2009
@@ -30,6 +30,50 @@
     /// </version>
     public class StandardAnalyzer : Analyzer
     {
+        /// <summary>An array containing some common English words that are usually not
+        /// useful for searching. 
+        /// </summary>
+        public static readonly System.String[] STOP_WORDS;
+
+        private static bool defaultReplaceInvalidAcronym;
+
+        static StandardAnalyzer()
+        {
+            STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS;
+
+            // Default to true (fixed the bug), unless the system property is set
+            string v = SupportClass.AppSettings.Get("Lucene.Net.Analysis.Standard.StandardAnalyzer.replaceInvalidAcronym", "true");
+            if (v.Equals("true"))
+                defaultReplaceInvalidAcronym = true;
+            else
+                defaultReplaceInvalidAcronym = false;
+        }
+
+        /// <summary>
+        /// see https://issues.apache.org/jira/browse/LUCENE-1068
+        /// </summary>
+        /// <returns>true if new instances of StandardTokenizer will replace mischaracterized acronyms</returns>
+        /// <deprecated>this will be removed (hardwired to true) in 3.0</deprecated>
+        public static bool GetDefaultReplaceInvalidAcronym()
+        {
+            return defaultReplaceInvalidAcronym;
+        }
+
+        /// <summary>
+        /// set to true to have new instances of StandardTokenizer
+        /// replace mischaracerized acronyms by default.  Set to 
+        /// false to preserve the previous (before 2.4) buggy behavior.
+        /// Alternatively, set the system property 
+        /// Lucene.Net.Analysis.Standard.StandardAnalyzer.replaceInvalidAcronym
+        /// to false.
+        /// </summary>
+        /// <param name="replaceInvalidAcronym"/>
+        /// <deprecated>this will be removed (hardwired to true) in 3.0</deprecated>
+        public static void SetDefaultReplaceInvalidAcronym(bool replaceInvalidAcronym)
+        {
+            defaultReplaceInvalidAcronym = replaceInvalidAcronym;
+        }
+
         private System.Collections.Hashtable stopSet;
 		
 		/// <summary> Specifies whether deprecated acronyms should be replaced with HOST type.
@@ -40,13 +84,8 @@
 		/// 
 		/// See https://issues.apache.org/jira/browse/LUCENE-1068
 		/// </deprecated>
-		private bool replaceInvalidAcronym = false;
-		
-        /// <summary>An array containing some common English words that are usually not
-        /// useful for searching. 
-        /// </summary>
-        public static readonly System.String[] STOP_WORDS;
-		
+		private bool replaceInvalidAcronym = defaultReplaceInvalidAcronym;
+        
         /// <summary>Builds an analyzer with the default stop words ({@link #STOP_WORDS}). </summary>
         public StandardAnalyzer() : this(STOP_WORDS)
         {
@@ -219,7 +258,8 @@
 		/// 
 		/// See https://issues.apache.org/jira/browse/LUCENE-1068
 		/// </returns>
-		public virtual bool IsReplaceInvalidAcronym()
+        /// <deprecated>this will be removed (hardwired to true) in 3.0</deprecated>
+        public virtual bool IsReplaceInvalidAcronym()
 		{
 			return replaceInvalidAcronym;
 		}
@@ -229,13 +269,10 @@
 		/// 
 		/// See https://issues.apache.org/jira/browse/LUCENE-1068
 		/// </param>
-		public virtual void  SetReplaceInvalidAcronym(bool replaceInvalidAcronym)
+        /// <deprecated>this will be removed (hardwired to true) in 3.0</deprecated>
+        public virtual void SetReplaceInvalidAcronym(bool replaceInvalidAcronym)
 		{
 			this.replaceInvalidAcronym = replaceInvalidAcronym;
 		}
-		static StandardAnalyzer()
-		{
-			STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS;
-		}
 	}
 }
\ No newline at end of file



Mime
View raw message