lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From busc...@apache.org
Subject svn commit: r799953 [2/4] - in /lucene/java/trunk: ./ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/ co...
Date Sat, 01 Aug 2009 22:52:35 GMT
Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java Sat Aug  1 22:52:32 2009
@@ -22,6 +22,9 @@
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
 import java.text.BreakIterator;
 
 /**
@@ -32,46 +35,62 @@
 public class ThaiWordFilter extends TokenFilter {
   
   private BreakIterator breaker = null;
-  private Token thaiToken = null;
   
+  private TermAttribute termAtt;
+  private OffsetAttribute offsetAtt;
+    
+  private State thaiState = null;
+
   public ThaiWordFilter(TokenStream input) {
     super(input);
     breaker = BreakIterator.getWordInstance(new Locale("th"));
+    termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+    offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
   }
   
-  public Token next(final Token reusableToken) throws IOException {
-    assert reusableToken != null;
-    if (thaiToken != null) {
+  public final boolean incrementToken() throws IOException {
+    if (thaiState != null) {
       int start = breaker.current();
       int end = breaker.next();
       if (end != BreakIterator.DONE) {
-        reusableToken.reinit(thaiToken, thaiToken.termBuffer(), start, end - start);
-        reusableToken.setStartOffset(thaiToken.startOffset()+start);
-        reusableToken.setEndOffset(thaiToken.startOffset()+end);
-        return reusableToken;
+        restoreState(thaiState);
+        termAtt.setTermBuffer(termAtt.termBuffer(), start, end - start);
+        offsetAtt.setOffset(offsetAtt.startOffset() + start, offsetAtt.startOffset() + end);
+        return true;
       }
-      thaiToken = null;
+      thaiState = null;
     }
 
-    Token nextToken = input.next(reusableToken);
-    if (nextToken == null || nextToken.termLength() == 0) {
-      return null;
-    }
+    if (input.incrementToken() == false || termAtt.termLength() == 0)
+      return false;
 
-    String text = nextToken.term();
+    String text = termAtt.term();
     if (UnicodeBlock.of(text.charAt(0)) != UnicodeBlock.THAI) {
-      nextToken.setTermBuffer(text.toLowerCase());
-      return nextToken;
+      termAtt.setTermBuffer(text.toLowerCase());
+      return true;
     }
+    
+    thaiState = captureState();
 
-    thaiToken = (Token) nextToken.clone();
     breaker.setText(text);
     int end = breaker.next();
     if (end != BreakIterator.DONE) {
-      nextToken.setTermBuffer(text, 0, end);
-      nextToken.setEndOffset(nextToken.startOffset() + end);
-      return nextToken;
+      termAtt.setTermBuffer(text, 0, end);
+      offsetAtt.setOffset(offsetAtt.startOffset(), offsetAtt.startOffset() + end);
+      return true;
     }
-    return null;
+    return false;
+  }
+  
+  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
+   * not be overridden. Delegates to the backwards compatibility layer. */
+  public final Token next(final Token reusableToken) throws java.io.IOException {
+    return super.next(reusableToken);
+  }
+
+  /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should
+   * not be overridden. Delegates to the backwards compatibility layer. */
+  public final Token next() throws java.io.IOException {
+    return super.next();
   }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java Sat Aug  1 22:52:32 2009
@@ -17,18 +17,12 @@
  * limitations under the License.
  */
 
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.StringReader;
 
 import junit.framework.TestCase;
 
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * Test the Arabic Normalization Filter
@@ -95,11 +89,10 @@
   private void check(final String input, final String expected) throws IOException {
     ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(new StringReader(input));
     ArabicNormalizationFilter filter = new ArabicNormalizationFilter(tokenStream);
-    final Token reusableToken = new Token();
-    Token nextToken = filter.next(reusableToken);
-    if (nextToken == null)
-      fail();
-    assertEquals(expected, nextToken.term());
+    TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);
+    
+    assertTrue(filter.incrementToken());
+    assertEquals(expected, termAtt.term());
     filter.close();
   }
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java Sat Aug  1 22:52:32 2009
@@ -17,17 +17,12 @@
  * limitations under the License.
  */
 
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.StringReader;
 
 import junit.framework.TestCase;
 
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * Test the Arabic Normalization Filter
@@ -118,11 +113,10 @@
   private void check(final String input, final String expected) throws IOException {
     ArabicLetterTokenizer tokenStream  = new ArabicLetterTokenizer(new StringReader(input));
     ArabicStemFilter filter = new ArabicStemFilter(tokenStream);
-    final Token reusableToken = new Token();
-    Token nextToken = filter.next(reusableToken);
-    if (nextToken == null)
-      fail();
-    assertEquals(expected, nextToken.term());
+    TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);
+    
+    assertTrue(filter.incrementToken());
+    assertEquals(expected, termAtt.term());
     filter.close();
   }
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java Sat Aug  1 22:52:32 2009
@@ -23,8 +23,8 @@
 import junit.framework.TestCase;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * Test the Brazilian Stem Filter, which only modifies the term text.
@@ -122,12 +122,10 @@
   private void check(final String input, final String expected) throws IOException {
     Analyzer analyzer = new BrazilianAnalyzer(); 
     TokenStream stream = analyzer.tokenStream("dummy", new StringReader(input));
-    final Token reusableToken = new Token();
-    Token nextToken = stream.next(reusableToken);
-    if (nextToken == null)
-      fail();
-    assertEquals(expected, nextToken.term());
-    assertTrue(stream.next(nextToken) == null);
+    TermAttribute text = (TermAttribute) stream.getAttribute(TermAttribute.class);
+    assertTrue(stream.incrementToken());
+    assertEquals(expected, text.term());
+    assertFalse(stream.incrementToken());
     stream.close();
   }
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java Sat Aug  1 22:52:32 2009
@@ -21,50 +21,49 @@
 import java.io.StringReader;
 
 import junit.framework.TestCase;
-import org.apache.lucene.analysis.Token;
+
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 
 public class TestCJKTokenizer extends TestCase{
+  
+  class TestToken {
+    String termText;
+    int start;
+    int end;
+    String type;
+  }
 
-  public Token newToken(String termText, int start, int end, int type) {
-    Token token = new Token(start, end);
-    token.setTermBuffer(termText);
-    token.setType(CJKTokenizer.TOKEN_TYPE_NAMES[type]);
+  public TestToken newToken(String termText, int start, int end, int type) {
+    TestToken token = new TestToken();
+    token.termText = termText;
+    token.type = CJKTokenizer.TOKEN_TYPE_NAMES[type];
+    token.start = start;
+    token.end = end;
     return token;
   }
 
-  public void checkCJKToken(final String str, final Token[] out_tokens) throws IOException {
+  public void checkCJKToken(final String str, final TestToken[] out_tokens) throws IOException {
     CJKTokenizer tokenizer = new CJKTokenizer(new StringReader(str));
-    int i = 0;
-    System.out.println("string[" + str + "]");
-    System.out.print("tokens[");
-    final Token reusableToken = new Token();
-    for (Token token = tokenizer.next(reusableToken) ;
-         token != null                               ; 
-         token = tokenizer.next(reusableToken)       ) {
-      if (token.term().equals(out_tokens[i].term()) 
-          && token.startOffset() == out_tokens[i].startOffset() 
-          && token.endOffset() == out_tokens[i].endOffset() 
-          && token.type().equals(out_tokens[i].type()) ) {
-        System.out.print( token.term() + " ");
-      }
-      else {
-        fail(token.term() + " (start: " + token.startOffset() 
-             + " end: " + token.endOffset() + " type: " + token.type() + ") != "
-             + out_tokens[i].term() + " (start: " + out_tokens[i].startOffset() 
-             + " end: " + out_tokens[i].endOffset() 
-             + " type: " + out_tokens[i].type() + ")");
-        break;
-      }
-      ++i;
+    TermAttribute termAtt = (TermAttribute) tokenizer.getAttribute(TermAttribute.class);
+    OffsetAttribute offsetAtt = (OffsetAttribute) tokenizer.getAttribute(OffsetAttribute.class);
+    TypeAttribute typeAtt = (TypeAttribute) tokenizer.getAttribute(TypeAttribute.class);
+    for (int i = 0; i < out_tokens.length; i++) {
+      assertTrue(tokenizer.incrementToken());
+      assertEquals(termAtt.term(), out_tokens[i].termText);
+      assertEquals(offsetAtt.startOffset(), out_tokens[i].start);
+      assertEquals(offsetAtt.endOffset(), out_tokens[i].end);
+      assertEquals(typeAtt.type(), out_tokens[i].type);
     }
-    System.out.println("]" + System.getProperty("line.separator"));
+    assertFalse(tokenizer.incrementToken());
   }
   
   public void testJa1() throws IOException {
     String str = "\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341";
        
-    Token[] out_tokens = { 
+    TestToken[] out_tokens = { 
       newToken("\u4e00\u4e8c", 0, 2, CJKTokenizer.DOUBLE_TOKEN_TYPE), 
       newToken("\u4e8c\u4e09", 1, 3, CJKTokenizer.DOUBLE_TOKEN_TYPE),
       newToken("\u4e09\u56db", 2, 4, CJKTokenizer.DOUBLE_TOKEN_TYPE),
@@ -81,7 +80,7 @@
   public void testJa2() throws IOException {
     String str = "\u4e00 \u4e8c\u4e09\u56db \u4e94\u516d\u4e03\u516b\u4e5d \u5341";
        
-    Token[] out_tokens = { 
+    TestToken[] out_tokens = { 
       newToken("\u4e00", 0, 1, CJKTokenizer.DOUBLE_TOKEN_TYPE), 
       newToken("\u4e8c\u4e09", 2, 4, CJKTokenizer.DOUBLE_TOKEN_TYPE),
       newToken("\u4e09\u56db", 3, 5, CJKTokenizer.DOUBLE_TOKEN_TYPE),
@@ -97,7 +96,7 @@
   public void testC() throws IOException {
     String str = "abc defgh ijklmn opqrstu vwxy z";
        
-    Token[] out_tokens = { 
+    TestToken[] out_tokens = { 
       newToken("abc", 0, 3, CJKTokenizer.SINGLE_TOKEN_TYPE), 
       newToken("defgh", 4, 9, CJKTokenizer.SINGLE_TOKEN_TYPE),
       newToken("ijklmn", 10, 16, CJKTokenizer.SINGLE_TOKEN_TYPE),
@@ -111,7 +110,7 @@
   public void testMix() throws IOException {
     String str = "\u3042\u3044\u3046\u3048\u304aabc\u304b\u304d\u304f\u3051\u3053";
        
-    Token[] out_tokens = { 
+    TestToken[] out_tokens = { 
       newToken("\u3042\u3044", 0, 2, CJKTokenizer.DOUBLE_TOKEN_TYPE), 
       newToken("\u3044\u3046", 1, 3, CJKTokenizer.DOUBLE_TOKEN_TYPE),
       newToken("\u3046\u3048", 2, 4, CJKTokenizer.DOUBLE_TOKEN_TYPE),
@@ -128,7 +127,7 @@
   public void testMix2() throws IOException {
     String str = "\u3042\u3044\u3046\u3048\u304aab\u3093c\u304b\u304d\u304f\u3051 \u3053";
        
-    Token[] out_tokens = { 
+    TestToken[] out_tokens = { 
       newToken("\u3042\u3044", 0, 2, CJKTokenizer.DOUBLE_TOKEN_TYPE), 
       newToken("\u3044\u3046", 1, 3, CJKTokenizer.DOUBLE_TOKEN_TYPE),
       newToken("\u3046\u3048", 2, 4, CJKTokenizer.DOUBLE_TOKEN_TYPE),
@@ -147,7 +146,7 @@
   public void testSingleChar() throws IOException {
     String str = "\u4e00";
        
-    Token[] out_tokens = { 
+    TestToken[] out_tokens = { 
       newToken("\u4e00", 0, 1, CJKTokenizer.DOUBLE_TOKEN_TYPE), 
     };
     checkCJKToken(str, out_tokens);

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java Sat Aug  1 22:52:32 2009
@@ -22,7 +22,7 @@
 
 import junit.framework.TestCase;
 
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 
 
 public class TestChineseTokenizer extends TestCase
@@ -34,12 +34,12 @@
 
         int correctStartOffset = 0;
         int correctEndOffset = 1;
-        final Token reusableToken = new Token();
-        for (Token nextToken = tokenizer.next(reusableToken); nextToken != null; nextToken = tokenizer.next(reusableToken)) {
-            assertEquals(correctStartOffset, nextToken.startOffset());
-            assertEquals(correctEndOffset, nextToken.endOffset());
-            correctStartOffset++;
-            correctEndOffset++;
+        OffsetAttribute offsetAtt = (OffsetAttribute) tokenizer.getAttribute(OffsetAttribute.class);
+        while (tokenizer.incrementToken()) {
+          assertEquals(correctStartOffset, offsetAtt.startOffset());
+          assertEquals(correctEndOffset, offsetAtt.endOffset());
+          correctStartOffset++;
+          correctEndOffset++;
         }
     }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java Sat Aug  1 22:52:32 2009
@@ -31,15 +31,14 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
-import org.apache.lucene.analysis.Token;
+import junit.framework.TestCase;
+
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
-import org.apache.lucene.analysis.compound.CompoundWordTokenFilterBase;
-import org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilter;
-import org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter;
 import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
-
-import junit.framework.TestCase;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 public class TestCompoundWordTokenFilter extends TestCase {
   private static String[] locations = {
@@ -155,16 +154,18 @@
 
   private void assertFiltersTo(TokenFilter tf, String[] s, int[] startOffset,
       int[] endOffset, int[] posIncr) throws Exception {
-    final Token reusableToken = new Token();
+    TermAttribute termAtt = (TermAttribute) tf.getAttribute(TermAttribute.class);
+    OffsetAttribute offsetAtt = (OffsetAttribute) tf.getAttribute(OffsetAttribute.class);
+    PositionIncrementAttribute posIncAtt = (PositionIncrementAttribute) tf.getAttribute(PositionIncrementAttribute.class);
+    
     for (int i = 0; i < s.length; ++i) {
-      Token nextToken = tf.next(reusableToken);
-      assertNotNull(nextToken);
-      assertEquals(s[i], nextToken.term());
-      assertEquals(startOffset[i], nextToken.startOffset());
-      assertEquals(endOffset[i], nextToken.endOffset());
-      assertEquals(posIncr[i], nextToken.getPositionIncrement());
+      assertTrue(tf.incrementToken());
+      assertEquals(s[i], termAtt.term());
+      assertEquals(startOffset[i], offsetAtt.startOffset());
+      assertEquals(endOffset[i], offsetAtt.endOffset());
+      assertEquals(posIncr[i], posIncAtt.getPositionIncrement());
     }
-    assertNull(tf.next(reusableToken));
+    assertFalse(tf.incrementToken());
   }
 
   private void getHyphenationPatternFileContents() {

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java Sat Aug  1 22:52:32 2009
@@ -22,8 +22,8 @@
 import junit.framework.TestCase;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * Test the CzechAnalyzer
@@ -39,13 +39,12 @@
 
   private void assertAnalyzesTo(Analyzer a, String input, String[] output) throws Exception {
     TokenStream ts = a.tokenStream("dummy", new StringReader(input));
-    final Token reusableToken = new Token();
+    TermAttribute text = (TermAttribute) ts.getAttribute(TermAttribute.class);
     for (int i=0; i<output.length; i++) {
-      Token nextToken = ts.next(reusableToken);
-      assertNotNull(nextToken);
-      assertEquals(nextToken.term(), output[i]);
+      assertTrue(ts.incrementToken());
+      assertEquals(text.term(), output[i]);
     }
-    assertNull(ts.next(reusableToken));
+    assertFalse(ts.incrementToken());
     ts.close();
   }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Sat Aug  1 22:52:32 2009
@@ -26,8 +26,8 @@
 
 import junit.framework.TestCase;
 
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * Test the German stemmer. The stemming algorithm is known to work less 
@@ -68,11 +68,9 @@
   private void check(final String input, final String expected) throws IOException {
     StandardTokenizer tokenStream = new StandardTokenizer(new StringReader(input));
     GermanStemFilter filter = new GermanStemFilter(tokenStream);
-    final Token reusableToken = new Token();
-    Token nextToken = filter.next(reusableToken);
-    if (nextToken == null)
-      fail();
-    assertEquals(expected, nextToken.term());
+    TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);
+    assertTrue(filter.incrementToken());
+    assertEquals(expected, termAtt.term());
     filter.close();
   }
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/el/GreekAnalyzerTest.java Sat Aug  1 22:52:32 2009
@@ -18,11 +18,11 @@
 
 import java.io.StringReader;
 
+import junit.framework.TestCase;
+
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
-
-import junit.framework.TestCase;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 
 /**
@@ -41,13 +41,12 @@
 	 */
 	private void assertAnalyzesTo(Analyzer a, String input, String[] output) throws Exception {
 		TokenStream ts = a.tokenStream("dummy", new StringReader(input));
-                final Token reusableToken = new Token();
+		TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
 		for (int i=0; i<output.length; i++) {
-		        Token nextToken = ts.next(reusableToken);
-			assertNotNull(nextToken);
-			assertEquals(nextToken.term(), output[i]);
+			assertTrue(ts.incrementToken());
+			assertEquals(termAtt.term(), output[i]);
 		}
-		assertNull(ts.next(reusableToken));
+		assertFalse(ts.incrementToken());
 		ts.close();
 	}
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestElision.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestElision.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestElision.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestElision.java Sat Aug  1 22:52:32 2009
@@ -26,10 +26,10 @@
 
 import junit.framework.TestCase;
 
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * 
@@ -52,9 +52,9 @@
   private List filtre(TokenFilter filter) {
     List tas = new ArrayList();
     try {
-      final Token reusableToken = new Token();
-      for (Token nextToken = filter.next(reusableToken); nextToken != null; nextToken = filter.next(reusableToken)) {
-        tas.add(nextToken.term());
+      TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);
+      while (filter.incrementToken()) {
+        tas.add(termAtt.term());
       }
     } catch (IOException e) {
       e.printStackTrace();

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java Sat Aug  1 22:52:32 2009
@@ -59,8 +59,8 @@
 import junit.framework.TestCase;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * Test case for FrenchAnalyzer.
@@ -76,13 +76,12 @@
 
 		TokenStream ts = a.tokenStream("dummy", new StringReader(input));
 
-                final Token reusableToken = new Token();
+		TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
 		for (int i = 0; i < output.length; i++) {
-			Token nextToken = ts.next(reusableToken);
-			assertNotNull(nextToken);
-			assertEquals(nextToken.term(), output[i]);
+			assertTrue(ts.incrementToken());
+			assertEquals(termAtt.term(), output[i]);
 		}
-		assertNull(ts.next(reusableToken));
+		assertFalse(ts.incrementToken());
 		ts.close();
 	}
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestEmptyTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestEmptyTokenStream.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestEmptyTokenStream.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestEmptyTokenStream.java Sat Aug  1 22:52:32 2009
@@ -17,20 +17,19 @@
  * limitations under the License.
  */
 
-import junit.framework.TestCase;
-
 import java.io.IOException;
 
+import junit.framework.TestCase;
+
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Token;
 
 public class TestEmptyTokenStream extends TestCase {
 
   public void test() throws IOException {
     TokenStream ts = new EmptyTokenStream();
-    assertNull(ts.next());
+    assertFalse(ts.incrementToken());
     ts.reset();
-    assertNull(ts.next(new Token()));
+    assertFalse(ts.incrementToken());
   }
 
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Sat Aug  1 22:52:32 2009
@@ -17,9 +17,9 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 import java.io.StringReader;
 
@@ -67,58 +67,57 @@
 
   public void testFrontUnigram() throws Exception {
     EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 1, 1);
-    final Token reusableToken = new Token();
-    Token nextToken = tokenizer.next(reusableToken);
-    assertEquals("(a,0,1)", nextToken.toString());
-    assertNull(tokenizer.next(reusableToken));
+    TermAttribute termAtt = (TermAttribute) tokenizer.addAttribute(TermAttribute.class);
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(a,0,1)", termAtt.toString());
+    assertFalse(tokenizer.incrementToken());
   }
 
   public void testBackUnigram() throws Exception {
     EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.BACK, 1, 1);
-    final Token reusableToken = new Token();
-    Token nextToken = tokenizer.next(reusableToken);
-    assertEquals("(e,4,5)", nextToken.toString());
-    assertNull(tokenizer.next(reusableToken));
+    TermAttribute termAtt = (TermAttribute) tokenizer.addAttribute(TermAttribute.class);
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(e,4,5)", termAtt.toString());
+    assertFalse(tokenizer.incrementToken());
   }
 
   public void testOversizedNgrams() throws Exception {
     EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 6, 6);
-    assertNull(tokenizer.next(new Token()));
+    assertFalse(tokenizer.incrementToken());
   }
 
   public void testFrontRangeOfNgrams() throws Exception {
     EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
-    final Token reusableToken = new Token();
-    Token nextToken = tokenizer.next(reusableToken);
-    assertEquals("(a,0,1)", nextToken.toString());
-    nextToken = tokenizer.next(reusableToken);
-    assertEquals("(ab,0,2)", nextToken.toString());
-    nextToken = tokenizer.next(reusableToken);
-    assertEquals("(abc,0,3)", nextToken.toString());
-    assertNull(tokenizer.next(reusableToken));
+    TermAttribute termAtt = (TermAttribute) tokenizer.addAttribute(TermAttribute.class);
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(a,0,1)", termAtt.toString());
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(ab,0,2)", termAtt.toString());
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(abc,0,3)", termAtt.toString());
+    assertFalse(tokenizer.incrementToken());
   }
 
   public void testBackRangeOfNgrams() throws Exception {
     EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.BACK, 1, 3);
-    final Token reusableToken = new Token();
-    Token nextToken = tokenizer.next(reusableToken);
-    assertEquals("(e,4,5)", nextToken.toString());
-    nextToken = tokenizer.next(reusableToken);
-    assertEquals("(de,3,5)", nextToken.toString());
-    nextToken = tokenizer.next(reusableToken);
-    assertEquals("(cde,2,5)", nextToken.toString());
-    assertNull(tokenizer.next(reusableToken));
+    TermAttribute termAtt = (TermAttribute) tokenizer.addAttribute(TermAttribute.class);
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(e,4,5)", termAtt.toString());
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(de,3,5)", termAtt.toString());
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(cde,2,5)", termAtt.toString());
+    assertFalse(tokenizer.incrementToken());
   }
   
   public void testSmallTokenInStream() throws Exception {
     input = new WhitespaceTokenizer(new StringReader("abc de fgh"));
     EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 3, 3);
-    final Token reusableToken = new Token();
-    Token nextToken = tokenizer.next(reusableToken);
-    assertEquals("(abc,0,3)", nextToken.toString());
-    nextToken = tokenizer.next(reusableToken);
-    assertNotNull(nextToken);
-    assertEquals("(fgh,0,3)", nextToken.toString());
-    assertNull(tokenizer.next(reusableToken));
+    TermAttribute termAtt = (TermAttribute) tokenizer.addAttribute(TermAttribute.class);
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(abc,0,3)", termAtt.toString());
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(fgh,0,3)", termAtt.toString());
+    assertFalse(tokenizer.incrementToken());
   }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java Sat Aug  1 22:52:32 2009
@@ -17,10 +17,11 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.Token;
 
 import java.io.StringReader;
 
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
 import junit.framework.TestCase;
 
 /**
@@ -65,46 +66,46 @@
 
   public void testFrontUnigram() throws Exception {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 1, 1);
-    final Token reusableToken = new Token();
-    Token nextToken = tokenizer.next(reusableToken);
-    assertEquals("(a,0,1)", nextToken.toString());
-    assertNull(tokenizer.next(reusableToken));
+    TermAttribute termAtt = (TermAttribute) tokenizer.addAttribute(TermAttribute.class);
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(a,0,1)", termAtt.toString());
+    assertFalse(tokenizer.incrementToken());
   }
 
   public void testBackUnigram() throws Exception {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.BACK, 1, 1);
-    final Token reusableToken = new Token();
-    Token nextToken = tokenizer.next(reusableToken);
-    assertEquals("(e,4,5)", nextToken.toString());
-    assertNull(tokenizer.next(reusableToken));
+    TermAttribute termAtt = (TermAttribute) tokenizer.addAttribute(TermAttribute.class);
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(e,4,5)", termAtt.toString());
+    assertFalse(tokenizer.incrementToken());
   }
 
   public void testOversizedNgrams() throws Exception {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 6, 6);
-    assertNull(tokenizer.next(new Token()));
+    assertFalse(tokenizer.incrementToken());
   }
 
   public void testFrontRangeOfNgrams() throws Exception {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 1, 3);
-    final Token reusableToken = new Token();
-    Token nextToken = tokenizer.next(reusableToken);
-    assertEquals("(a,0,1)", nextToken.toString());
-    nextToken = tokenizer.next(reusableToken);
-    assertEquals("(ab,0,2)", nextToken.toString());
-    nextToken = tokenizer.next(reusableToken);
-    assertEquals("(abc,0,3)", nextToken.toString());
-    assertNull(tokenizer.next(reusableToken));
+    TermAttribute termAtt = (TermAttribute) tokenizer.addAttribute(TermAttribute.class);
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(a,0,1)", termAtt.toString());
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(ab,0,2)", termAtt.toString());
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(abc,0,3)", termAtt.toString());
+    assertFalse(tokenizer.incrementToken());
   }
 
   public void testBackRangeOfNgrams() throws Exception {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.BACK, 1, 3);
-    final Token reusableToken = new Token();
-    Token nextToken = tokenizer.next(reusableToken);
-    assertEquals("(e,4,5)", nextToken.toString());
-    nextToken = tokenizer.next(reusableToken);
-    assertEquals("(de,3,5)", nextToken.toString());
-    nextToken = tokenizer.next(reusableToken);
-    assertEquals("(cde,2,5)", nextToken.toString());
-    assertNull(tokenizer.next(reusableToken));
+    TermAttribute termAtt = (TermAttribute) tokenizer.addAttribute(TermAttribute.class);
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(e,4,5)", termAtt.toString());
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(de,3,5)", termAtt.toString());
+    assertTrue(tokenizer.incrementToken());
+    assertEquals("(cde,2,5)", termAtt.toString());
+    assertFalse(tokenizer.incrementToken());
   }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java Sat Aug  1 22:52:32 2009
@@ -17,12 +17,12 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
+import java.io.IOException;
 import java.io.StringReader;
-import java.util.ArrayList;
 
 import junit.framework.TestCase;
 
@@ -31,7 +31,6 @@
  */
 public class NGramTokenFilterTest extends TestCase {
     private TokenStream input;
-    private ArrayList tokens = new ArrayList();
     
     public void setUp() {
         input = new WhitespaceTokenizer(new StringReader("abcde"));
@@ -57,79 +56,56 @@
         assertTrue(gotException);
     }
 
+    private void checkStream(TokenStream stream, String[] exp) throws IOException {
+      TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);
+      for (int i = 0; i < exp.length; i++) {
+        assertTrue(stream.incrementToken());
+        assertEquals(exp[i], termAtt.toString());
+      }
+      assertFalse(stream.incrementToken());
+    }
+    
     public void testUnigrams() throws Exception {
       NGramTokenFilter filter = new NGramTokenFilter(input, 1, 1);
-
-      final Token reusableToken = new Token();
-        for (Token nextToken = filter.next(reusableToken); nextToken != null; nextToken = filter.next(reusableToken)) {
-            tokens.add(nextToken.toString());
-//          System.out.println(token.term());
-//          System.out.println(token);
-//          Thread.sleep(1000);
-        }
-
-        assertEquals(5, tokens.size());
-        ArrayList exp = new ArrayList();
-        exp.add("(a,0,1)"); exp.add("(b,1,2)"); exp.add("(c,2,3)"); exp.add("(d,3,4)"); exp.add("(e,4,5)");
-        assertEquals(exp, tokens);
+      String[] exp = new String[] {
+        "(a,0,1)", "(b,1,2)", "(c,2,3)", "(d,3,4)", "(e,4,5)"
+      };
+      
+      checkStream(filter, exp);
     }
 
     public void testBigrams() throws Exception {
       NGramTokenFilter filter = new NGramTokenFilter(input, 2, 2);
-      final Token reusableToken = new Token();
-        for (Token nextToken = filter.next(reusableToken); nextToken != null; nextToken = filter.next(reusableToken)) {
-            tokens.add(nextToken.toString());
-//          System.out.println(token.term());
-//          System.out.println(token);
-//          Thread.sleep(1000);
-        }
-
-        assertEquals(4, tokens.size());
-        ArrayList exp = new ArrayList();
-        exp.add("(ab,0,2)"); exp.add("(bc,1,3)"); exp.add("(cd,2,4)"); exp.add("(de,3,5)");
-        assertEquals(exp, tokens);
+      String[] exp = new String[] {
+          "(ab,0,2)", "(bc,1,3)", "(cd,2,4)", "(de,3,5)"
+        };
+        
+      checkStream(filter, exp);
     }
 
     public void testNgrams() throws Exception {
       NGramTokenFilter filter = new NGramTokenFilter(input, 1, 3);
-      final Token reusableToken = new Token();
-        for (Token nextToken = filter.next(reusableToken); nextToken != null; nextToken = filter.next(reusableToken)) {
-            tokens.add(nextToken.toString());
-//          System.out.println(token.term());
-//          System.out.println(token);
-//          Thread.sleep(1000);
-        }
-
-        assertEquals(12, tokens.size());
-        ArrayList exp = new ArrayList();
-        exp.add("(a,0,1)"); exp.add("(b,1,2)"); exp.add("(c,2,3)"); exp.add("(d,3,4)"); exp.add("(e,4,5)");
-        exp.add("(ab,0,2)"); exp.add("(bc,1,3)"); exp.add("(cd,2,4)"); exp.add("(de,3,5)");
-        exp.add("(abc,0,3)"); exp.add("(bcd,1,4)"); exp.add("(cde,2,5)");
-        assertEquals(exp, tokens);
+      String[] exp = new String[] {
+          "(a,0,1)", "(b,1,2)", "(c,2,3)", "(d,3,4)", "(e,4,5)",
+          "(ab,0,2)", "(bc,1,3)", "(cd,2,4)", "(de,3,5)",
+          "(abc,0,3)", "(bcd,1,4)", "(cde,2,5)"
+      };
+        
+      checkStream(filter, exp);
     }
 
     public void testOversizedNgrams() throws Exception {
       NGramTokenFilter filter = new NGramTokenFilter(input, 6, 7);
-      final Token reusableToken = new Token();
-        for (Token nextToken = filter.next(reusableToken); nextToken != null; nextToken = filter.next(reusableToken)) {
-            tokens.add(nextToken.toString());
-//          System.out.println(token.term());
-//          System.out.println(token);
-//          Thread.sleep(1000);
-        }
-
-        assertTrue(tokens.isEmpty());
+      assertFalse(filter.incrementToken());
     }
     
     public void testSmallTokenInStream() throws Exception {
       input = new WhitespaceTokenizer(new StringReader("abc de fgh"));
       NGramTokenFilter filter = new NGramTokenFilter(input, 3, 3);
-      final Token reusableToken = new Token();
-      Token nextToken = filter.next(reusableToken);
-      assertEquals("(abc,0,3)", nextToken.toString());
-      nextToken = filter.next(reusableToken);
-      assertNotNull(nextToken);
-      assertEquals("(fgh,0,3)", nextToken.toString());
-      assertNull(filter.next(reusableToken));
+      String[] exp = new String[] {
+          "(abc,0,3)", "(fgh,0,3)"
+        };
+        
+      checkStream(filter, exp);
     }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java Sat Aug  1 22:52:32 2009
@@ -17,10 +17,12 @@
  * limitations under the License.
  */
 
-import org.apache.lucene.analysis.Token;
 
+import java.io.IOException;
 import java.io.StringReader;
-import java.util.ArrayList;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 import junit.framework.TestCase;
 
@@ -29,7 +31,6 @@
  */
 public class NGramTokenizerTest extends TestCase {
     private StringReader input;
-    private ArrayList tokens = new ArrayList();
     
     public void setUp() {
         input = new StringReader("abcde");
@@ -54,69 +55,48 @@
         }
         assertTrue(gotException);
     }
+    
+    private void checkStream(TokenStream stream, String[] exp) throws IOException {
+      TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class);
+      for (int i = 0; i < exp.length; i++) {
+        assertTrue(stream.incrementToken());
+        assertEquals(exp[i], termAtt.toString());
+      }
+      assertFalse(stream.incrementToken());
+    }
 
     public void testUnigrams() throws Exception {
         NGramTokenizer tokenizer = new NGramTokenizer(input, 1, 1);
         
-        final Token reusableToken = new Token();
-        for (Token nextToken = tokenizer.next(reusableToken); nextToken != null; nextToken = tokenizer.next(reusableToken)) {
-          tokens.add(nextToken.toString());
-//        System.out.println(token.term());
-//        System.out.println(token);
-//        Thread.sleep(1000);
-      }
-
-        assertEquals(5, tokens.size());
-        ArrayList exp = new ArrayList();
-        exp.add("(a,0,1)"); exp.add("(b,1,2)"); exp.add("(c,2,3)"); exp.add("(d,3,4)"); exp.add("(e,4,5)");
-        assertEquals(exp, tokens);
+        String[] exp = new String[] {
+            "(a,0,1)", "(b,1,2)", "(c,2,3)", "(d,3,4)", "(e,4,5)"
+          };
+          
+        checkStream(tokenizer, exp);
     }
 
     public void testBigrams() throws Exception {
         NGramTokenizer tokenizer = new NGramTokenizer(input, 2, 2);
-        final Token reusableToken = new Token();
-        for (Token nextToken = tokenizer.next(reusableToken); nextToken != null; nextToken = tokenizer.next(reusableToken)) {
-          tokens.add(nextToken.toString());
-//        System.out.println(token.term());
-//        System.out.println(token);
-//        Thread.sleep(1000);
-      }
-
-        assertEquals(4, tokens.size());
-        ArrayList exp = new ArrayList();
-        exp.add("(ab,0,2)"); exp.add("(bc,1,3)"); exp.add("(cd,2,4)"); exp.add("(de,3,5)");
-        assertEquals(exp, tokens);
+        String[] exp = new String[] {
+            "(ab,0,2)", "(bc,1,3)", "(cd,2,4)", "(de,3,5)"
+          };
+          
+        checkStream(tokenizer, exp);
     }
 
     public void testNgrams() throws Exception {
         NGramTokenizer tokenizer = new NGramTokenizer(input, 1, 3);
-        final Token reusableToken = new Token();
-        for (Token nextToken = tokenizer.next(reusableToken); nextToken != null; nextToken = tokenizer.next(reusableToken)) {
-          tokens.add(nextToken.toString());
-//        System.out.println(token.term());
-//        System.out.println(token);
-//        Thread.sleep(1000);
-      }
-
-        assertEquals(12, tokens.size());
-        ArrayList exp = new ArrayList();
-        exp.add("(a,0,1)"); exp.add("(b,1,2)"); exp.add("(c,2,3)"); exp.add("(d,3,4)"); exp.add("(e,4,5)");
-        exp.add("(ab,0,2)"); exp.add("(bc,1,3)"); exp.add("(cd,2,4)"); exp.add("(de,3,5)");
-        exp.add("(abc,0,3)"); exp.add("(bcd,1,4)"); exp.add("(cde,2,5)");
-        assertEquals(exp, tokens);
+        String[] exp = new String[] {
+            "(a,0,1)", "(b,1,2)", "(c,2,3)", "(d,3,4)", "(e,4,5)",
+            "(ab,0,2)", "(bc,1,3)", "(cd,2,4)", "(de,3,5)",
+            "(abc,0,3)", "(bcd,1,4)", "(cde,2,5)"
+        };
+          
+        checkStream(tokenizer, exp);
     }
 
     public void testOversizedNgrams() throws Exception {
         NGramTokenizer tokenizer = new NGramTokenizer(input, 6, 7);
-
-        final Token reusableToken = new Token();
-        for (Token nextToken = tokenizer.next(reusableToken); nextToken != null; nextToken = tokenizer.next(reusableToken)) {
-          tokens.add(nextToken.toString());
-//        System.out.println(token.term());
-//        System.out.println(token);
-//        Thread.sleep(1000);
-      }
-
-        assertTrue(tokens.isEmpty());
+        assertFalse(tokenizer.incrementToken());
     }
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java Sat Aug  1 22:52:32 2009
@@ -23,8 +23,8 @@
 import junit.framework.TestCase;
 
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * Test the Dutch Stem Filter, which only modifies the term text.
@@ -121,12 +121,10 @@
   private void check(final String input, final String expected) throws IOException {
     Analyzer analyzer = new DutchAnalyzer(); 
     TokenStream stream = analyzer.tokenStream("dummy", new StringReader(input));
-    final Token reusableToken = new Token();
-    Token nextToken = stream.next(reusableToken);
-    if (nextToken == null)
-      fail();
-    assertEquals(expected, nextToken.term());
-    assertTrue(stream.next(nextToken) == null);
+    TermAttribute text = (TermAttribute) stream.getAttribute(TermAttribute.class);
+    assertTrue(stream.incrementToken());
+    assertEquals(expected, text.term());
+    assertFalse(stream.incrementToken());
     stream.close();
   }
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java Sat Aug  1 22:52:32 2009
@@ -18,7 +18,6 @@
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.index.Payload;
@@ -65,7 +64,7 @@
     assertTermEquals("lazy", filter, "JJ".getBytes("UTF-8"));
     assertTermEquals("brown", filter, "JJ".getBytes("UTF-8"));
     assertTermEquals("dogs", filter, "NN".getBytes("UTF-8"));
-    assertTrue(filter.next(new Token()) == null);
+    assertFalse(filter.incrementToken());
   }
 
 
@@ -106,10 +105,11 @@
   }
 
   void assertTermEquals(String expected, TokenStream stream, byte[] expectPay) throws Exception {
-    Token tok = new Token();
-    assertTrue(stream.next(tok) != null);
-    assertEquals(expected, tok.term());
-    Payload payload = tok.getPayload();
+    TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
+    PayloadAttribute payloadAtt = (PayloadAttribute) stream.getAttribute(PayloadAttribute.class);
+    assertTrue(stream.incrementToken());
+    assertEquals(expected, termAtt.term());
+    Payload payload = payloadAtt.getPayload();
     if (payload != null) {
       assertTrue(payload.length() + " does not equal: " + expectPay.length, payload.length() == expectPay.length);
       for (int i = 0; i < expectPay.length; i++) {

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java Sat Aug  1 22:52:32 2009
@@ -17,10 +17,12 @@
  */
 
 import junit.framework.TestCase;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -44,36 +46,44 @@
 
     NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(new WhitespaceTokenizer(new StringReader(test))), 3, "D");
     boolean seenDogs = false;
-    final Token reusableToken = new Token();
-    for (Token nextToken = nptf.next(reusableToken); nextToken != null; nextToken = nptf.next(reusableToken)) {
-      if (nextToken.term().equals("dogs")){
+    TermAttribute termAtt = (TermAttribute) nptf.getAttribute(TermAttribute.class);
+    TypeAttribute typeAtt = (TypeAttribute) nptf.getAttribute(TypeAttribute.class);
+    PayloadAttribute payloadAtt = (PayloadAttribute) nptf.getAttribute(PayloadAttribute.class);
+    while (nptf.incrementToken()) {
+      if (termAtt.term().equals("dogs")) {
         seenDogs = true;
-        assertTrue(nextToken.type() + " is not equal to " + "D", nextToken.type().equals("D") == true);
-        assertTrue("nextToken.getPayload() is null and it shouldn't be", nextToken.getPayload() != null);
-        byte [] bytes = nextToken.getPayload().getData();//safe here to just use the bytes, otherwise we should use offset, length
-        assertTrue(bytes.length + " does not equal: " + nextToken.getPayload().length(), bytes.length == nextToken.getPayload().length());
-        assertTrue(nextToken.getPayload().getOffset() + " does not equal: " + 0, nextToken.getPayload().getOffset() == 0);
+        assertTrue(typeAtt.type() + " is not equal to " + "D", typeAtt.type().equals("D") == true);
+        assertTrue("payloadAtt.getPayload() is null and it shouldn't be", payloadAtt.getPayload() != null);
+        byte [] bytes = payloadAtt.getPayload().getData();//safe here to just use the bytes, otherwise we should use offset, length
+        assertTrue(bytes.length + " does not equal: " + payloadAtt.getPayload().length(), bytes.length == payloadAtt.getPayload().length());
+        assertTrue(payloadAtt.getPayload().getOffset() + " does not equal: " + 0, payloadAtt.getPayload().getOffset() == 0);
         float pay = PayloadHelper.decodeFloat(bytes);
         assertTrue(pay + " does not equal: " + 3, pay == 3);
       } else {
-        assertTrue(nextToken.type() + " is not null and it should be", nextToken.type().equals("word"));
+        assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().equals("word"));
       }
     }
     assertTrue(seenDogs + " does not equal: " + true, seenDogs == true);
   }
 
-  private class WordTokenFilter extends TokenFilter {
+  private final class WordTokenFilter extends TokenFilter {
+    private TermAttribute termAtt;
+    private TypeAttribute typeAtt;
+    
     private WordTokenFilter(TokenStream input) {
       super(input);
+      termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+      typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
     }
-
-    public Token next(final Token reusableToken) throws IOException {
-      assert reusableToken != null;
-      Token nextToken = input.next(reusableToken);
-      if (nextToken != null && nextToken.term().equals("dogs")) {
-        nextToken.setType("D");
+    
+    public boolean incrementToken() throws IOException {
+      if (input.incrementToken()) {
+        if (termAtt.term().equals("dogs"))
+          typeAtt.setType("D");
+        return true;
+      } else {
+        return false;
       }
-      return nextToken;
     }
   }
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java Sat Aug  1 22:52:32 2009
@@ -17,8 +17,9 @@
  */
 
 import junit.framework.TestCase;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
 import org.apache.lucene.index.Payload;
 
 import java.io.IOException;
@@ -43,16 +44,17 @@
 
     TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(new WhitespaceTokenizer(new StringReader(test)));
     int count = 0;
-    final Token reusableToken = new Token();
-    for (Token nextToken = nptf.next(reusableToken); nextToken != null; nextToken = nptf.next(reusableToken)) {
-      assertTrue("nextToken is null and it shouldn't be", nextToken != null);
-      Payload pay = nextToken.getPayload();
+    PayloadAttribute payloadAtt = (PayloadAttribute) nptf.getAttribute(PayloadAttribute.class);
+    OffsetAttribute offsetAtt = (OffsetAttribute) nptf.getAttribute(OffsetAttribute.class);
+    
+    while (nptf.incrementToken()) {
+      Payload pay = payloadAtt.getPayload();
       assertTrue("pay is null and it shouldn't be", pay != null);
       byte [] data = pay.getData();
       int start = PayloadHelper.decodeInt(data, 0);
-      assertTrue(start + " does not equal: " + nextToken.startOffset(), start == nextToken.startOffset());
+      assertTrue(start + " does not equal: " + offsetAtt.startOffset(), start == offsetAtt.startOffset());
       int end = PayloadHelper.decodeInt(data, 4);
-      assertTrue(end + " does not equal: " + nextToken.endOffset(), end == nextToken.endOffset());
+      assertTrue(end + " does not equal: " + offsetAtt.endOffset(), end == offsetAtt.endOffset());
       count++;
     }
     assertTrue(count + " does not equal: " + 10, count == 10);

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java Sat Aug  1 22:52:32 2009
@@ -19,8 +19,10 @@
 import junit.framework.TestCase;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -45,32 +47,39 @@
 
     TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(new WhitespaceTokenizer(new StringReader(test))));
     int count = 0;
-    final Token reusableToken = new Token();
-    for (Token nextToken = nptf.next(reusableToken); nextToken != null; nextToken = nptf.next(reusableToken)) {
-      assertTrue(nextToken.type() + " is not null and it should be", nextToken.type().equals(String.valueOf(Character.toUpperCase(nextToken.termBuffer()[0]))));
-      assertTrue("nextToken.getPayload() is null and it shouldn't be", nextToken.getPayload() != null);
-      String type = new String(nextToken.getPayload().getData(), "UTF-8");
+    TermAttribute termAtt = (TermAttribute) nptf.getAttribute(TermAttribute.class);
+    TypeAttribute typeAtt = (TypeAttribute) nptf.getAttribute(TypeAttribute.class);
+    PayloadAttribute payloadAtt = (PayloadAttribute) nptf.getAttribute(PayloadAttribute.class);
+    
+    while (nptf.incrementToken()) {
+      assertTrue(typeAtt.type() + " is not null and it should be", typeAtt.type().equals(String.valueOf(Character.toUpperCase(termAtt.termBuffer()[0]))));
+      assertTrue("nextToken.getPayload() is null and it shouldn't be", payloadAtt.getPayload() != null);
+      String type = new String(payloadAtt.getPayload().getData(), "UTF-8");
       assertTrue("type is null and it shouldn't be", type != null);
-      assertTrue(type + " is not equal to " + nextToken.type(), type.equals(nextToken.type()) == true);
+      assertTrue(type + " is not equal to " + typeAtt.type(), type.equals(typeAtt.type()) == true);
       count++;
     }
+
     assertTrue(count + " does not equal: " + 10, count == 10);
   }
 
-  private class WordTokenFilter extends TokenFilter {
+  private final class WordTokenFilter extends TokenFilter {
+    private TermAttribute termAtt;
+    private TypeAttribute typeAtt;
+    
     private WordTokenFilter(TokenStream input) {
       super(input);
+      termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+      typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
     }
 
-
-
-    public Token next(final Token reusableToken) throws IOException {
-      assert reusableToken != null;
-      Token nextToken = input.next(reusableToken);
-      if (nextToken != null) {
-        nextToken.setType(String.valueOf(Character.toUpperCase(nextToken.termBuffer()[0])));
+    public boolean incrementToken() throws IOException {
+      if (input.incrementToken()) {
+        typeAtt.setType(String.valueOf(Character.toUpperCase(termAtt.termBuffer()[0])));
+        return true;
+      } else {
+        return false;
       }
-      return nextToken;
     }
   }
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java Sat Aug  1 22:52:32 2009
@@ -19,9 +19,9 @@
 
 import java.io.StringReader;
 
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.LuceneTestCase;
 
 public class TestReverseStringFilter extends LuceneTestCase {
@@ -29,13 +29,18 @@
     TokenStream stream = new WhitespaceTokenizer(
         new StringReader("Do have a nice day"));     // 1-4 length string
     ReverseStringFilter filter = new ReverseStringFilter(stream);
-    final Token reusableToken = new Token();
-    assertEquals("oD", filter.next(reusableToken).term());
-    assertEquals("evah", filter.next(reusableToken).term());
-    assertEquals("a", filter.next(reusableToken).term());
-    assertEquals("ecin", filter.next(reusableToken).term());
-    assertEquals("yad", filter.next(reusableToken).term());
-    assertNull(filter.next(reusableToken));
+    TermAttribute text = (TermAttribute) filter.getAttribute(TermAttribute.class);
+    assertTrue(filter.incrementToken());
+    assertEquals("oD", text.term());
+    assertTrue(filter.incrementToken());
+    assertEquals("evah", text.term());
+    assertTrue(filter.incrementToken());
+    assertEquals("a", text.term());
+    assertTrue(filter.incrementToken());
+    assertEquals("ecin", text.term());
+    assertTrue(filter.incrementToken());
+    assertEquals("yad", text.term());
+    assertFalse(filter.incrementToken());
   }
 
   public void testReverseString() throws Exception {

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java Sat Aug  1 22:52:32 2009
@@ -26,8 +26,8 @@
 
 import junit.framework.TestCase;
 
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 /**
  * Test case for RussianAnalyzer.
@@ -77,26 +77,21 @@
                 sampleUnicode,
                 RussianCharsets.UnicodeRussian);
 
-        final Token reusableToken = new Token();
-        final Token reusableSampleToken = new Token();
-        Token nextToken;
-        Token nextSampleToken;
+        TermAttribute text = (TermAttribute) in.getAttribute(TermAttribute.class);
+        TermAttribute sampleText = (TermAttribute) sample.getAttribute(TermAttribute.class);
+
         for (;;)
         {
-            nextToken = in.next(reusableToken);
-
-            if (nextToken == null)
-            {
-                break;
-            }
+          if (in.incrementToken() == false)
+            break;
 
-            nextSampleToken = sample.next(reusableSampleToken);
+            boolean nextSampleToken = sample.incrementToken();
             assertEquals(
                 "Unicode",
-                nextToken.term(),
-                nextSampleToken == null
+                text.term(),
+                nextSampleToken == false
                 ? null
-                : nextSampleToken.term());
+                : sampleText.term());
         }
 
         inWords.close();
@@ -118,29 +113,22 @@
                 sampleKOI8,
                 RussianCharsets.KOI8);
 
-        final Token reusableToken = new Token();
-        final Token reusableSampleToken = new Token();
-        Token nextToken;
-        Token nextSampleToken;
+        TermAttribute text = (TermAttribute) in.getAttribute(TermAttribute.class);
+        TermAttribute sampleText = (TermAttribute) sample.getAttribute(TermAttribute.class);
+
         for (;;)
         {
-            nextToken = in.next(reusableToken);
+          if (in.incrementToken() == false)
+            break;
 
-            if (nextToken == null)
-            {
-                break;
-            }
-
-            nextSampleToken = sample.next(reusableSampleToken);
+            boolean nextSampleToken = sample.incrementToken();
             assertEquals(
                 "KOI8",
-                nextToken.term(),
-                nextSampleToken == null
+                text.term(),
+                nextSampleToken == false
                 ? null
-                : nextSampleToken.term());
-
+                : sampleText.term());
         }
-
         inWordsKOI8.close();
         sampleKOI8.close();
     }
@@ -159,27 +147,21 @@
                 sample1251,
                 RussianCharsets.CP1251);
 
-        final Token reusableToken = new Token();
-        final Token reusableSampleToken = new Token();
-        Token nextToken;
-        Token nextSampleToken;
+        TermAttribute text = (TermAttribute) in.getAttribute(TermAttribute.class);
+        TermAttribute sampleText = (TermAttribute) sample.getAttribute(TermAttribute.class);
+
         for (;;)
         {
-          nextToken = in.next(reusableToken);
+          if (in.incrementToken() == false)
+            break;
 
-            if (nextToken == null)
-            {
-                break;
-            }
-
-            nextSampleToken = sample.next(reusableSampleToken);
+            boolean nextSampleToken = sample.incrementToken();
             assertEquals(
                 "1251",
-                nextToken.term(),
-                nextSampleToken == null
+                text.term(),
+                nextSampleToken == false
                 ? null
-                : nextSampleToken.term());
-
+                : sampleText.term());
         }
 
         inWords1251.close();
@@ -192,10 +174,13 @@
         RussianAnalyzer ra = new RussianAnalyzer();
         TokenStream stream = ra.tokenStream("", reader);
 
-        final Token reusableToken = new Token();
+        TermAttribute termText = (TermAttribute) stream.getAttribute(TermAttribute.class);
         try {
-            assertEquals("text", stream.next(reusableToken).term());
-            assertNotNull("RussianAnalyzer's tokenizer skips numbers from input text", stream.next(reusableToken));
+            assertTrue(stream.incrementToken());
+            assertEquals("text", termText.term());
+            assertTrue(stream.incrementToken());
+            assertEquals("RussianAnalyzer's tokenizer skips numbers from input text", "1000", termText.term());
+            assertFalse(stream.incrementToken());
         }
         catch (IOException e)
         {

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Sat Aug  1 22:52:32 2009
@@ -18,10 +18,14 @@
  */
 
 import java.io.StringReader;
+
 import junit.framework.TestCase;
+
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 /**
  * Test case for ThaiAnalyzer, modified from TestFrenchAnalyzer
@@ -70,19 +74,20 @@
 		throws Exception {
 
 		TokenStream ts = a.tokenStream("dummy", new StringReader(input));
-                final Token reusableToken = new Token();
+		TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
+		OffsetAttribute offsetAtt = (OffsetAttribute) ts.getAttribute(OffsetAttribute.class);
+		TypeAttribute typeAtt = (TypeAttribute) ts.getAttribute(TypeAttribute.class);
 		for (int i = 0; i < output.length; i++) {
-			Token nextToken = ts.next(reusableToken);
-			assertNotNull(nextToken);
-			assertEquals(nextToken.term(), output[i]);
+			assertTrue(ts.incrementToken());
+			assertEquals(termAtt.term(), output[i]);
 			if (startOffsets != null)
-				assertEquals(nextToken.startOffset(), startOffsets[i]);
+				assertEquals(offsetAtt.startOffset(), startOffsets[i]);
 			if (endOffsets != null)
-				assertEquals(nextToken.endOffset(), endOffsets[i]);
+				assertEquals(offsetAtt.endOffset(), endOffsets[i]);
 			if (types != null)
-				assertEquals(nextToken.type(), types[i]);
+				assertEquals(typeAtt.type(), types[i]);
 		}
-		assertNull(ts.next(reusableToken));
+		assertFalse(ts.incrementToken());
 		ts.close();
 	}
 	

Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (original)
+++ lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java Sat Aug  1 22:52:32 2009
@@ -20,8 +20,10 @@
 import java.io.IOException;
 import java.io.Reader;
 
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 /**
  * Tokenizes input text into sentences.
@@ -29,7 +31,7 @@
  * The output tokens can then be broken into words with {@link WordTokenFilter}
  * </p>
  */
-public class SentenceTokenizer extends Tokenizer {
+public final class SentenceTokenizer extends Tokenizer {
 
   /**
    * End of sentence punctuation: 。,!?;,!?;
@@ -39,12 +41,19 @@
   private final StringBuffer buffer = new StringBuffer();
 
   private int tokenStart = 0, tokenEnd = 0;
+  
+  private TermAttribute termAtt;
+  private OffsetAttribute offsetAtt;
+  private TypeAttribute typeAtt;
 
   public SentenceTokenizer(Reader reader) {
     super(reader);
+    termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+    offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
+    typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
   }
 
-  public Token next(final Token reusableToken) throws IOException {
+  public boolean incrementToken() throws IOException {
     buffer.setLength(0);
     int ci;
     char ch, pch;
@@ -83,11 +92,12 @@
       }
     }
     if (buffer.length() == 0)
-      return null;
+      return false;
     else {
-      reusableToken.clear();
-      reusableToken.reinit(buffer.toString(), input.correctOffset(tokenStart), input.correctOffset(tokenEnd), "sentence");
-      return reusableToken;
+      termAtt.setTermBuffer(buffer.toString());
+      offsetAtt.setOffset(input.correctOffset(tokenStart), input.correctOffset(tokenEnd));
+      typeAtt.setType("sentence");
+      return true;
     }
   }
 

Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java (original)
+++ lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java Sat Aug  1 22:52:32 2009
@@ -20,7 +20,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.cn.smart.hhmm.HHMMSegmenter;
 import org.apache.lucene.analysis.cn.smart.hhmm.SegToken;
 import org.apache.lucene.analysis.cn.smart.hhmm.SegTokenFilter;
@@ -37,11 +36,11 @@
   /**
    * Segment a sentence into words with {@link HHMMSegmenter}
    * 
-   * @param sentenceToken sentence {@link Token}
+   * @param sentence input sentence
+   * @param startOffset start offset of sentence
    * @return {@link List} of {@link SegToken}
    */
-  public List segmentSentence(Token sentenceToken) {
-    String sentence = sentenceToken.term();
+  public List segmentSentence(String sentence, int startOffset) {
 
     List segTokenList = hhmmSegmenter.process(sentence);
 
@@ -49,25 +48,25 @@
 
     // tokens from sentence, excluding WordType.SENTENCE_BEGIN and WordType.SENTENCE_END
     for (int i = 1; i < segTokenList.size() - 1; i++) {
-      result.add(convertSegToken((SegToken) segTokenList.get(i), sentence,
-          sentenceToken.startOffset(), "word"));
+      result.add(convertSegToken((SegToken) segTokenList.get(i), sentence, startOffset));
     }
     return result;
 
   }
 
   /**
-   * Convert a {@link SegToken} to a Lucene {@link Token}
+   * Process a {@link SegToken} so that it is ready for indexing.
+   * 
+   * This method calculates offsets and normalizes the token with {@link SegTokenFilter}.
    * 
    * @param st input {@link SegToken}
    * @param sentence associated Sentence
    * @param sentenceStartOffset offset into sentence
-   * @param type token type, default is word
-   * @return Lucene {@link Token}
+   * @return Lucene {@link SegToken}
    */
-  public Token convertSegToken(SegToken st, String sentence,
-      int sentenceStartOffset, String type) {
-    Token result;
+  public SegToken convertSegToken(SegToken st, String sentence,
+      int sentenceStartOffset) {
+
     switch (st.wordType) {
       case WordType.STRING:
       case WordType.NUMBER:
@@ -81,9 +80,8 @@
     }
 
     st = tokenFilter.filter(st);
-
-    result = new Token(st.charArray, 0, st.charArray.length, st.startOffset
-        + sentenceStartOffset, st.endOffset + sentenceStartOffset);
-    return result;
+    st.startOffset += sentenceStartOffset;
+    st.endOffset += sentenceStartOffset;
+    return st;
   }
 }

Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java Sat Aug  1 22:52:32 2009
@@ -21,20 +21,27 @@
 import java.util.Iterator;
 import java.util.List;
 
-import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.cn.smart.hhmm.SegToken;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 /**
  * A {@link TokenFilter} that breaks sentences into words.
  */
-public class WordTokenFilter extends TokenFilter {
+public final class WordTokenFilter extends TokenFilter {
 
   private WordSegmenter wordSegmenter;
 
   private Iterator tokenIter;
 
   private List tokenBuffer;
+  
+  private TermAttribute termAtt;
+  private OffsetAttribute offsetAtt;
+  private TypeAttribute typeAtt;
 
   /**
    * Construct a new WordTokenizer.
@@ -44,32 +51,34 @@
   public WordTokenFilter(TokenStream in) {
     super(in);
     this.wordSegmenter = new WordSegmenter();
+    termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+    offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
+    typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
   }
-
-  public Token next(final Token reusableSentenceToken) throws IOException {
-    if (tokenIter != null && tokenIter.hasNext())
-      return (Token) tokenIter.next();
-    else {
-      Token nextToken = input.next(reusableSentenceToken);
-      if (processNextSentence(nextToken)) {
-        return (Token) tokenIter.next();
-      } else
-        return null;
-    }
-  }
-
-  /**
-   * Process the next input sentence, placing tokens into tokenBuffer
-   * 
-   * @param reusableSentenceToken input sentence
-   * @return true if more tokens were placed into tokenBuffer.
-   * @throws IOException
-   */
-  private boolean processNextSentence(final Token reusableSentenceToken) throws IOException {
-    if (reusableSentenceToken == null)
-      return false;
-    tokenBuffer = wordSegmenter.segmentSentence(reusableSentenceToken);
-    tokenIter = tokenBuffer.iterator();
-    return tokenBuffer != null && tokenIter.hasNext();
+  
+  public boolean incrementToken() throws IOException {   
+    if (tokenIter == null || !tokenIter.hasNext()) {
+      // there are no remaining tokens from the current sentence... are there more sentences?
+      if (input.incrementToken()) {
+        // a new sentence is available: process it.
+        tokenBuffer = wordSegmenter.segmentSentence(termAtt.term(), offsetAtt.startOffset());
+        tokenIter = tokenBuffer.iterator();
+        /* 
+         * it should not be possible to have a sentence with 0 words, check just in case.
+         * returning EOS isn't the best either, but its the behavior of the original code.
+         */
+        if (!tokenIter.hasNext())
+          return false;
+      } else {
+        return false; // no more sentences, end of stream!
+      }
+    } 
+    
+    // There are remaining tokens from the current sentence, return the next one. 
+    SegToken nextWord = (SegToken) tokenIter.next();
+    termAtt.setTermBuffer(nextWord.charArray, 0, nextWord.charArray.length);
+    offsetAtt.setOffset(nextWord.startOffset, nextWord.endOffset);
+    typeAtt.setType("word");
+    return true;
   }
 }

Modified: lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/TestSmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/TestSmartChineseAnalyzer.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/TestSmartChineseAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/TestSmartChineseAnalyzer.java Sat Aug  1 22:52:32 2009
@@ -29,6 +29,9 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 
 public class TestSmartChineseAnalyzer extends TestCase {
   
@@ -108,22 +111,23 @@
   public void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[])
   throws Exception {
 
-  TokenStream ts = a.tokenStream("dummy", new StringReader(input));
-          final Token reusableToken = new Token();
-  for (int i = 0; i < output.length; i++) {
-      Token nextToken = ts.next(reusableToken);
-      assertNotNull(nextToken);
-      assertEquals(nextToken.term(), output[i]);
+    TokenStream ts = a.tokenStream("dummy", new StringReader(input));
+    TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
+    OffsetAttribute offsetAtt = (OffsetAttribute) ts.getAttribute(OffsetAttribute.class);
+    TypeAttribute typeAtt = (TypeAttribute) ts.getAttribute(TypeAttribute.class);
+    for (int i = 0; i < output.length; i++) {
+      assertTrue(ts.incrementToken());
+      assertEquals(termAtt.term(), output[i]);
       if (startOffsets != null)
-          assertEquals(nextToken.startOffset(), startOffsets[i]);
+        assertEquals(offsetAtt.startOffset(), startOffsets[i]);
       if (endOffsets != null)
-          assertEquals(nextToken.endOffset(), endOffsets[i]);
+        assertEquals(offsetAtt.endOffset(), endOffsets[i]);
       if (types != null)
-          assertEquals(nextToken.type(), types[i]);
+        assertEquals(typeAtt.type(), types[i]);
+    }
+    assertFalse(ts.incrementToken());
+    ts.close();
   }
-  assertNull(ts.next(reusableToken));
-  ts.close();
-}
 
 public void assertAnalyzesTo(Analyzer a, String input, String[] output) throws Exception {
   assertAnalyzesTo(a, input, output, null, null, null);

Modified: lucene/java/trunk/contrib/collation/src/java/org/apache/lucene/collation/CollationKeyFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/collation/src/java/org/apache/lucene/collation/CollationKeyFilter.java?rev=799953&r1=799952&r2=799953&view=diff
==============================================================================
--- lucene/java/trunk/contrib/collation/src/java/org/apache/lucene/collation/CollationKeyFilter.java (original)
+++ lucene/java/trunk/contrib/collation/src/java/org/apache/lucene/collation/CollationKeyFilter.java Sat Aug  1 22:52:32 2009
@@ -21,6 +21,7 @@
 import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.util.IndexableBinaryStringTools;
 
 import java.io.IOException;
@@ -73,8 +74,9 @@
  *   {@link ICUCollationKeyFilter} on the query side, or vice versa.
  * </p>
  */
-public class CollationKeyFilter extends TokenFilter {
+public final class CollationKeyFilter extends TokenFilter {
   private Collator collator = null;
+  private TermAttribute termAtt;
 
   /**
    * @param input Source token stream
@@ -83,25 +85,26 @@
   public CollationKeyFilter(TokenStream input, Collator collator) {
     super(input);
     this.collator = collator;
+    termAtt = (TermAttribute) addAttribute(TermAttribute.class);
   }
 
-  public final Token next(final Token reusableToken) throws IOException {
-    assert reusableToken != null;
-    Token nextToken = input.next(reusableToken);
-    if (nextToken != null) {
-      char[] termBuffer = nextToken.termBuffer();
-      String termText = new String(termBuffer, 0, nextToken.termLength());
+  public boolean incrementToken() throws IOException {
+    if (input.incrementToken()) {
+      char[] termBuffer = termAtt.termBuffer();
+      String termText = new String(termBuffer, 0, termAtt.termLength());
       byte[] collationKey = collator.getCollationKey(termText).toByteArray();
       ByteBuffer collationKeyBuf = ByteBuffer.wrap(collationKey);
       int encodedLength
         = IndexableBinaryStringTools.getEncodedLength(collationKeyBuf);
       if (encodedLength > termBuffer.length) {
-        nextToken.resizeTermBuffer(encodedLength);
+        termAtt.resizeTermBuffer(encodedLength);
       }
-      nextToken.setTermLength(encodedLength);
-      CharBuffer wrappedTermBuffer = CharBuffer.wrap(nextToken.termBuffer());
+      termAtt.setTermLength(encodedLength);
+      CharBuffer wrappedTermBuffer = CharBuffer.wrap(termAtt.termBuffer());
       IndexableBinaryStringTools.encode(collationKeyBuf, wrappedTermBuffer);
+      return true;
+    } else {
+      return false;
     }
-    return nextToken;
   }
 }



Mime
View raw message