lucene-java-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mikemcc...@apache.org
Subject svn commit: r887525 - in /lucene/java/branches/flex_1458/src: java/org/apache/lucene/index/ java/org/apache/lucene/index/codecs/ java/org/apache/lucene/index/codecs/preflex/ java/org/apache/lucene/index/codecs/standard/ test/org/apache/lucene/ test/org...
Date Sat, 05 Dec 2009 10:47:48 GMT
Author: mikemccand
Date: Sat Dec  5 10:47:47 2009
New Revision: 887525

URL: http://svn.apache.org/viewvc?rev=887525&view=rev
Log:
LUCENE-2111: fix bug in 'flex API on non-flex index' layer

Modified:
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CompoundFileReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacySegmentMergeInfo.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyTerms.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
    lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsIndexReader.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java
    lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CompoundFileReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CompoundFileReader.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CompoundFileReader.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CompoundFileReader.java
Sat Dec  5 10:47:47 2009
@@ -32,7 +32,8 @@
  * This class implements a directory, but is limited to only read operations.
  * Directory methods that would normally modify data throw an exception.
  */
-class CompoundFileReader extends Directory {
+// nocmmit -- made public
+public class CompoundFileReader extends Directory {
 
     private int readBufferSize;
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java Sat
Dec  5 10:47:47 2009
@@ -1127,8 +1127,8 @@
     public FieldsEnumWithBase(IndexReader reader, int base) throws IOException {
       this.base = base;
       length = reader.maxDoc();
-      deletedDocs = reader.getDeletedDocs();
-      fields = reader.fields().iterator();
+      deletedDocs = reader.getDeletedDocs(); 
+     fields = reader.fields().iterator();
     }
   }
 
@@ -1265,6 +1265,7 @@
     }
   }
 
+  // Exposes flex API, merged from flex API of sub-segments
   private final static class MultiFieldsEnum extends FieldsEnum {
     private final FieldMergeQueue queue;
 
@@ -1329,6 +1330,7 @@
     }
   }
 
+  // Exposes flex API, merged from flex API of sub-segments
   private static final class MultiTermsEnum extends TermsEnum {
     
     private final TermMergeQueue queue;
@@ -1641,8 +1643,9 @@
     }
   }
 
-
-  // Legacy API
+  // @deprecated This is pre-flex API
+  // Exposes pre-flex API by doing on-the-fly merging
+  // pre-flex API to each segment
   static class MultiTermEnum extends TermEnum {
     IndexReader topReader; // used for matching TermEnum to TermDocs
     private LegacySegmentMergeQueue queue;
@@ -1729,7 +1732,9 @@
     }
   }
 
-  // Legacy API
+  // @deprecated This is pre-flex API
+  // Exposes pre-flex API by doing on-the-fly merging
+  // pre-flex API to each segment
   static class MultiTermDocs implements TermDocs {
     IndexReader topReader;  // used for matching TermEnum to TermDocs
     protected IndexReader[] readers;
@@ -1881,7 +1886,9 @@
     }
   }
 
-  // Legacy API
+  // @deprecated This is pre-flex API
+  // Exposes pre-flex API by doing on-the-fly merging
+  // pre-flex API to each segment
   static class MultiTermPositions extends MultiTermDocs implements TermPositions {
     public MultiTermPositions(IndexReader topReader, IndexReader[] r, int[] s) {
       super(topReader,r,s);

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java Sat
Dec  5 10:47:47 2009
@@ -64,7 +64,8 @@
   static final String VECTORS_INDEX_EXTENSION = "tvx";
 
   /** Extension of compound file */
-  static final String COMPOUND_FILE_EXTENSION = "cfs";
+  // nocommit made public
+  public static final String COMPOUND_FILE_EXTENSION = "cfs";
 
   /** Extension of compound file for doc store files*/
   static final String COMPOUND_FILE_STORE_EXTENSION = "cfx";

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java Sat Dec
 5 10:47:47 2009
@@ -621,7 +621,7 @@
           // index (eg because it's doing deletes, or an NRT
           // reader is being opened) we ask the reader to
           // load its terms index.
-          sr.loadTermsIndex();
+          sr.loadTermsIndex(termsIndexDivisor);
         }
       }
 

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFields.java Sat
Dec  5 10:47:47 2009
@@ -40,8 +40,4 @@
     // nocommit
     return new LegacyTerms(r, field);
   }
-
-  public void close() throws IOException {
-    // nocommit
-  }
-}
\ No newline at end of file
+}

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyFieldsEnum.java
Sat Dec  5 10:47:47 2009
@@ -20,9 +20,9 @@
 import java.io.IOException;
 import org.apache.lucene.util.Bits;
 
-/** Implements new API (FieldsEnum/TermsEnum) on top of old
- *  API.  Used only for IndexReader impls outside Lucene's
- *  core. */
+/** Implements flex API (FieldsEnum/TermsEnum) on top of
+ *  pre-flex API.  Used only for IndexReader impls outside
+ *  Lucene's core. */
 class LegacyFieldsEnum extends FieldsEnum {
   private final IndexReader r;
   private TermEnum terms;
@@ -38,14 +38,6 @@
     terms = r.terms(t);
   }
 
-  /*
-  public boolean seek(String field) throws IOException {
-    this.field = field;
-    doSeek(new Term(field, ""));
-    return terms.term() != null && terms.term().field.equals(field);
-  }
-  */
-
   @Override
   public String next() throws IOException {
 
@@ -68,11 +60,6 @@
     return new LegacyTermsEnum(r, field);
   }
 
-  public void close() throws IOException {
-    terms.close();
-  }
-
-  // Emulates flex on top of legacy API
   static class LegacyTermsEnum extends TermsEnum {
     private final IndexReader r;
     private final String field;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacySegmentMergeInfo.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacySegmentMergeInfo.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacySegmentMergeInfo.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacySegmentMergeInfo.java
Sat Dec  5 10:47:47 2009
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 
+// @deprecated This is pre-flex API
 final class LegacySegmentMergeInfo {
   Term term;
   int base;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyTerms.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyTerms.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyTerms.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/LegacyTerms.java Sat Dec
 5 10:47:47 2009
@@ -20,9 +20,9 @@
 
 import java.io.IOException;
 
-/** Implements new API (FieldsEnum/TermsEnum) on top of old
- *  API.  Used only for IndexReader impls outside Lucene's
- *  core. */
+/** Implements flex API (FieldsEnum/TermsEnum) on top of
+ *  pre-flex API.  Used only for IndexReader impls outside
+ *  Lucene's core. */
 class LegacyTerms extends Terms {
 
   private final IndexReader r;
@@ -38,9 +38,6 @@
     return new LegacyFieldsEnum.LegacyTermsEnum(r, field);
   }
 
-  public void close() {
-  }
-
   @Override
   public TermRef.Comparator getTermComparator() {
     // Pre-flex indexes always sorted in UTF16 order

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java Sat
Dec  5 10:47:47 2009
@@ -1136,8 +1136,8 @@
   // sharing a segment that's still being merged.  This
   // method is not thread safe, and relies on the
   // synchronization in IndexWriter
-  void loadTermsIndex() throws IOException {
-    core.fields.loadTermsIndex();
+  void loadTermsIndex(int indexDivisor) throws IOException {
+    core.fields.loadTermsIndex(indexDivisor);
   }
 
   // for testing only

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Terms.java Sat Dec  5
10:47:47 2009
@@ -26,7 +26,6 @@
 
 public abstract class Terms {
 
-  // nocommit -- char[] or byte[] version?
   /** Returns an iterator that will step through all terms */
   public abstract TermsEnum iterator() throws IOException;
   

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/FieldsProducer.java
Sat Dec  5 10:47:47 2009
@@ -30,5 +30,5 @@
  */
 public abstract class FieldsProducer extends Fields {
   public abstract void close() throws IOException;
-  public abstract void loadTermsIndex() throws IOException;
+  public abstract void loadTermsIndex(int indexDivisor) throws IOException;
 }

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
Sat Dec  5 10:47:47 2009
@@ -33,27 +33,43 @@
 import org.apache.lucene.index.TermRef;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.CompoundFileReader;
 import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.FieldsProducer;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Bits;
 
+/** Exposes flex API on a pre-flex index, as a codec. */
 public class PreFlexFields extends FieldsProducer {
 
   // nocommit -- needed public by SegmentReader
-  public final TermInfosReader tis;
+  public TermInfosReader tis;
+  public final TermInfosReader tisNoIndex;
 
   // nocomit -- needed public by SR
   public final IndexInput freqStream;
   // nocomit -- needed public by SR
   public final IndexInput proxStream;
   final private FieldInfos fieldInfos;
+  private final SegmentInfo si;
   final TreeMap<String,FieldInfo> fields = new TreeMap<String,FieldInfo>();
+  private final Directory dir;
+  private final int readBufferSize;
+  private Directory cfsReader;
 
   PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize,
int indexDivisor)
     throws IOException {
-    tis = new TermInfosReader(dir, info.name, fieldInfos, readBufferSize, indexDivisor);
   
+
+    si = info;
+    TermInfosReader r = new TermInfosReader(dir, info.name, fieldInfos, readBufferSize, indexDivisor);
   
+    if (indexDivisor == -1) {
+      tisNoIndex = r;
+    } else {
+      tisNoIndex = null;
+      tis = r;
+    }
+    this.readBufferSize = readBufferSize;
     this.fieldInfos = fieldInfos;
 
     // make sure that all index files have been read or are kept open
@@ -76,6 +92,8 @@
     } else {
       proxStream = null;
     }
+
+    this.dir = dir;
   }
 
   static void files(Directory dir, SegmentInfo info, Collection<String> files) throws
IOException {
@@ -111,20 +129,58 @@
     }
   }
 
+  synchronized private TermInfosReader getTermsDict() {
+    if (tis != null) {
+      return tis;
+    } else {
+      return tisNoIndex;
+    }
+  }
+
   @Override
-  public void loadTermsIndex() throws IOException {
-    // nocommit -- todo
+  synchronized public void loadTermsIndex(int indexDivisor) throws IOException {
+    if (tis == null) {
+      Directory dir0;
+      if (si.getUseCompoundFile()) {
+        // In some cases, we were originally opened when CFS
+        // was not used, but then we are asked to open the
+        // terms reader with index, the segment has switched
+        // to CFS
+
+        // nocommit -- not clean that I open my own CFS
+        // reader here; caller should pass it in?
+        if (!(dir instanceof CompoundFileReader)) {
+          dir0 = cfsReader = new CompoundFileReader(dir, si.name + "." + IndexFileNames.COMPOUND_FILE_EXTENSION,
readBufferSize);
+        } else {
+          dir0 = dir;
+        }
+        dir0 = cfsReader;
+      } else {
+        dir0 = dir;
+      }
+
+      tis = new TermInfosReader(dir0, si.name, fieldInfos, readBufferSize, indexDivisor);
+    }
   }
 
   @Override
   public void close() throws IOException {
-    tis.close();
+    if (tis != null) {
+      tis.close();
+    }
+    if (tisNoIndex != null) {
+      tisNoIndex.close();
+    }
+    if (cfsReader != null) {
+      cfsReader.close();
+    }
   }
 
   private class Fields extends FieldsEnum {
     Iterator<FieldInfo> it;
     FieldInfo current;
-    private PreTermsEnum lastTermsEnum;
+    private SegmentTermEnum lastTermEnum;
+    private int fieldCount;
 
     public Fields() {
       it = fields.values().iterator();
@@ -133,6 +189,7 @@
     @Override
     public String next() {
       if (it.hasNext()) {
+        fieldCount++;
         current = it.next();
         return current.name;
       } else {
@@ -143,13 +200,17 @@
     @Override
     public TermsEnum terms() throws IOException {
       final PreTermsEnum terms;
-      if (lastTermsEnum != null) {
-        // Carry over SegmentTermsEnum
-        terms = new PreTermsEnum(current, lastTermsEnum.terms);
+      if (lastTermEnum != null) {
+        // Re-use SegmentTermEnum to avoid seeking for
+        // linear scan (done by merging)
+        terms = new PreTermsEnum(current, lastTermEnum);
       } else {
-        terms = new PreTermsEnum(current);
+        // If fieldCount is 1 then the terms enum can simply
+        // start at the start of the index (need not seek to
+        // the current field):
+        terms = new PreTermsEnum(current, fieldCount != 1);
+        lastTermEnum = terms.terms;
       }
-      lastTermsEnum = terms;
       return terms;
     }
   }
@@ -161,9 +222,9 @@
     }
 
     @Override
-    public TermsEnum iterator() {
+    public TermsEnum iterator() throws IOException {
       //System.out.println("pff.init create no context");
-      return new PreTermsEnum(fieldInfo);
+      return new PreTermsEnum(fieldInfo, true);
     }
 
     @Override
@@ -176,18 +237,36 @@
   private class PreTermsEnum extends TermsEnum {
     private SegmentTermEnum terms;
     private final FieldInfo fieldInfo;
-    private PreDocsEnum docsEnum;
+    private PreDocsEnum docsEnum; // nocommit -- unused
     private boolean skipNext;
     private TermRef current;
+    private final TermRef scratchTermRef = new TermRef();
 
-    PreTermsEnum(FieldInfo fieldInfo) {
+    // Pass needsSeek=false if the field is the very first
+    // field in the index -- this is used for linear scan of
+    // the index, eg when merging segments:
+    PreTermsEnum(FieldInfo fieldInfo, boolean needsSeek) throws IOException {
       this.fieldInfo = fieldInfo;
-      terms = tis.terms();
+      if (!needsSeek) {
+        terms = getTermsDict().terms();
+      } else {
+        terms = getTermsDict().terms(new Term(fieldInfo.name, ""));
+        skipNext = true;
+      }
     }
 
-    PreTermsEnum(FieldInfo fieldInfo, SegmentTermEnum terms) {
+    PreTermsEnum(FieldInfo fieldInfo, SegmentTermEnum terms) throws IOException {
       this.fieldInfo = fieldInfo;
-      this.terms = terms;
+      if (terms.term() == null || terms.term().field() != fieldInfo.name) {
+        terms = getTermsDict().terms(new Term(fieldInfo.name, ""));
+      } else {
+        // Carefully avoid seeking in the linear-scan case,
+        // because segment doesn't load/need the terms dict
+        // index during merging.  If the terms is already on
+        // our field, it must be because it had seeked to
+        // exhaustion on the last field
+        this.terms = terms;
+      }
       skipNext = true;
       if (Codec.DEBUG) {
         System.out.println("pff.terms.init field=" + fieldInfo.name);
@@ -215,15 +294,13 @@
       if (Codec.DEBUG) {
         System.out.println("pff.seek term=" + term);
       }
-      terms = tis.terms(new Term(fieldInfo.name, term.toString()));
+      terms = getTermsDict().terms(new Term(fieldInfo.name, term.toString()));
       final Term t = terms.term();
-      //System.out.println("  got to term=" + t  + " field eq?=" + (t.field() == fieldInfo.name)
+ " term eq?=" +
-      //term.equals(new TermRef(t.text())));
 
-      // nocommit -- reuse TermRef instance
       final TermRef tr;
       if (t != null) {
-        tr = new TermRef(t.text());
+        tr = scratchTermRef;
+        scratchTermRef.copy(t.text());
       } else {
         tr = null;
       }
@@ -245,8 +322,8 @@
       if (skipNext) {
         // nocommit -- is there a cleaner way?
         skipNext = false;
-        // nocommit -- reuse TermRef
-        current = new TermRef(terms.term().text());
+        scratchTermRef.copy(terms.term().text());
+        current = scratchTermRef;
         return current;
       }
       if (terms.next()) {
@@ -255,11 +332,11 @@
           System.out.println("pff.next term=" + t);
         }
         if (t.field() == fieldInfo.name) {
-          // nocommit -- reuse TermRef instance
           if (Codec.DEBUG) {
             System.out.println("  ok");
           }
-          current = new TermRef(t.text());
+          scratchTermRef.copy(t.text());
+          current = scratchTermRef;
           return current;
         } else {
           // Crossed into new field
@@ -298,16 +375,16 @@
     final private PrePositionsEnum prePos;
 
     PreDocsEnum(Bits skipDocs, Term t) throws IOException {
-      current = docs = new SegmentTermDocs(freqStream, skipDocs, tis, fieldInfos);
-      pos = new SegmentTermPositions(freqStream, proxStream, skipDocs, tis, fieldInfos);
+      current = docs = new SegmentTermDocs(freqStream, skipDocs, getTermsDict(), fieldInfos);
+      pos = new SegmentTermPositions(freqStream, proxStream, skipDocs, getTermsDict(), fieldInfos);
       prePos = new PrePositionsEnum(pos);
       docs.seek(t);
       pos.seek(t);
     }
 
     PreDocsEnum(Bits skipDocs, SegmentTermEnum te) throws IOException {
-      current = docs = new SegmentTermDocs(freqStream, skipDocs, tis, fieldInfos);
-      pos = new SegmentTermPositions(freqStream, proxStream, skipDocs, tis, fieldInfos);
+      current = docs = new SegmentTermDocs(freqStream, skipDocs, getTermsDict(), fieldInfos);
+      pos = new SegmentTermPositions(freqStream, proxStream, skipDocs, getTermsDict(), fieldInfos);
       prePos = new PrePositionsEnum(pos);
       docs.seek(te);
       pos.seek(te);

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/SimpleStandardTermsIndexReader.java
Sat Dec  5 10:47:47 2009
@@ -59,7 +59,7 @@
 public class SimpleStandardTermsIndexReader extends StandardTermsIndexReader {
 
   final private int totalIndexInterval;
-  final private int indexDivisor;
+  private int indexDivisor;
   final private int indexInterval;
 
   final private IndexInput in;
@@ -471,10 +471,11 @@
   }
 
   @Override
-  public void loadTermsIndex() throws IOException {
-
+  public void loadTermsIndex(int indexDivisor) throws IOException {
     if (!indexLoaded) {
 
+      this.indexDivisor = indexDivisor;
+
       // mxx
       if (Codec.DEBUG) {
         System.out.println(Thread.currentThread().getName() + ": sstir: load coreIndex on
demand");

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardDocsReader.java
Sat Dec  5 10:47:47 2009
@@ -296,7 +296,7 @@
 
         while(true) {
           if (count == docFreq) {
-            return NO_MORE_DOCS;
+            return doc = NO_MORE_DOCS;
           }
 
           count++;
@@ -499,8 +499,7 @@
         
         // Now, linear scan for the rest:
         do {
-          if (nextDoc() == NO_MORE_DOCS)
-            return NO_MORE_DOCS;
+          nextDoc();
         } while (target > doc);
 
         return doc;

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsDictReader.java
Sat Dec  5 10:47:47 2009
@@ -127,8 +127,11 @@
   }
 
   @Override
-  public void loadTermsIndex() throws IOException {
-    indexReader.loadTermsIndex();
+  public void loadTermsIndex(int indexDivisor) throws IOException {
+    // nocommit -- must handle case where segment has become
+    // a CFS since we originall opened; maybe Directory
+    // should be passed in?
+    indexReader.loadTermsIndex(indexDivisor);
   }
 
   @Override

Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsIndexReader.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsIndexReader.java
(original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardTermsIndexReader.java
Sat Dec  5 10:47:47 2009
@@ -63,7 +63,7 @@
 
   public abstract FieldReader getField(FieldInfo fieldInfo);
 
-  public abstract void loadTermsIndex() throws IOException;
+  public abstract void loadTermsIndex(int indexDivisor) throws IOException;
 
   public abstract void close() throws IOException;
 

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java Sat
Dec  5 10:47:47 2009
@@ -88,7 +88,7 @@
       }
 
       @Override
-      public void loadTermsIndex() {
+      public void loadTermsIndex(int indexDivisor) {
       }
     } 
 
@@ -586,11 +586,11 @@
       }
 
       @Override
-      public void loadTermsIndex() throws IOException {
+      public void loadTermsIndex(int indexDivisor) throws IOException {
         Iterator<FieldsProducer> it = codecs.values().iterator();
         while(it.hasNext()) {
           // nocommit -- catch exc and keep closing the rest?
-          it.next().loadTermsIndex();
+          it.next().loadTermsIndex(indexDivisor);
         }
       }
     }

Modified: lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=887525&r1=887524&r2=887525&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
(original)
+++ lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
Sat Dec  5 10:47:47 2009
@@ -645,4 +645,59 @@
   /* This was used in 2.9 to generate an index with compressed field:
   static final int BINARY_COMPRESSED_LENGTH = CompressionTools.compress(BINARY_TO_COMPRESS).length;
   */
+
+  private int countDocs(DocsEnum docs) throws IOException {
+    int count = 0;
+    while((docs.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
+      count ++;
+    }
+    return count;
+  }
+
+  // flex: test basics of TermsEnum api on non-flex index
+  public void testNextIntoWrongField() throws Exception {
+    for(int i=0;i<oldNames.length;i++) {
+      String dirName = "src/test/org/apache/lucene/index/index." + oldNames[i];
+      unzip(dirName, oldNames[i]);
+      String fullPath = fullDir(oldNames[i]);
+      Directory dir = FSDirectory.open(new File(fullPath));
+      IndexReader r = IndexReader.open(dir);
+      TermsEnum terms = r.fields().terms("content").iterator();
+      TermRef t = terms.next();
+      assertNotNull(t);
+
+      // content field only has term aaa:
+      assertEquals("aaa", t.toString());
+      assertNull(terms.next());
+
+      TermRef aaaTerm = new TermRef("aaa");
+
+      // should be found exactly
+      assertEquals(TermsEnum.SeekStatus.FOUND,
+                   terms.seek(aaaTerm));
+      assertEquals(35, countDocs(terms.docs(null)));
+      assertNull(terms.next());
+
+      // should hit end of field
+      assertEquals(TermsEnum.SeekStatus.END,
+                   terms.seek(new TermRef("bbb")));
+      assertNull(terms.next());
+
+      // should seek to aaa
+      assertEquals(TermsEnum.SeekStatus.NOT_FOUND,
+                   terms.seek(new TermRef("a")));
+      assertTrue(terms.term().termEquals(aaaTerm));
+      assertEquals(35, countDocs(terms.docs(null)));
+      assertNull(terms.next());
+
+      assertEquals(TermsEnum.SeekStatus.FOUND,
+                   terms.seek(aaaTerm));
+      assertEquals(35, countDocs(terms.docs(null)));
+      assertNull(terms.next());
+
+      r.close();
+      dir.close();
+      rmDir(oldNames[i]);
+    }
+  }
 }



Mime
View raw message