lucene-java-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Ankit Murarka <ankit.mura...@rancoretech.com>
Subject Stream Closed Exception and Lock Obtain Failed Exception while reading the file in chunks iteratively.
Date Sun, 01 Sep 2013 11:28:22 GMT
Hello.
I am struck in a problem and have been continously getting exception of 
/*Stream Closed and LockObtainFailedException..*/

I am trying to read the complete document line by line and once I have 
read 100000 lines I will add the doc to the writer and then again read 
the next 100000 lines and repeat the same process unless the complete 
file is read.

I cannot read the entire document line by line in 1 go due to memory 
constraints.

Can anyone kindly let me know what might be wrong with this code:

The */StreamCLOSED/ *exception occurs when I try to add doc to writer 
for iterations>1..In first iteration everything goes fine. /*Lock 
exception */follows streamclosed exception..

Code Snippet:

package com.iorg;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LiveIndexWriterConfig;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergePolicy.OneMerge;
import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

import com.rancore.CustomAnalyzerForCaseSensitive;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.util.Date;
import java.util.Iterator;

public class MainClass2 {
   public static void main(String[] args) {

     String indexPath = args[0];  //Place where indexes will be created
     String docsPath=args[1];    //Place where the files are kept.

    final File docDir = new File(docsPath);
    if (!docDir.exists() || !docDir.canRead()) {
       System.out.println("Document directory '" 
+docDir.getAbsolutePath()+ "' does not exist or is not readable, please 
check the path");
       System.exit(1);
     }
     Date start = new Date();
    try {
       System.out.println("Indexing to directory ONLY '" + indexPath + 
"'..."+docsPath);
      Directory dir = FSDirectory.open(new File(indexPath));
      Analyzer analyzer=new 
CustomAnalyzerForCaseSensitive(Version.LUCENE_44);
      IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_44, 
analyzer);
      iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);

       if(args[2].trim().equalsIgnoreCase("OverAll")){
           System.out.println("IN");
           indexDocs(docDir,true,dir,iwc);
       }
       Date end = new Date();
      System.out.println(end.getTime() - start.getTime() + " total 
milliseconds");

     } catch (IOException e) {
       System.out.println(" caught a " + e.getClass() +
        "\n with message: " + e.getMessage());
     }
     catch(Exception e)
     {
         e.printStackTrace();
     }
  }

   //Over All
static void indexDocs(File file,boolean flag,Directory 
dir,IndexWriterConfig iwc)
   throws IOException {
       FileInputStream fis = null;
  if (file.canRead()) {
     if (file.isDirectory())
     {
      String[] files = file.list();
      System.out.println("size of list is "  + files.length);
       if (files != null) {
         for (int i = 0; i < files.length; i++) {
             System.out.println("Invoked for  "  +  i  +  "and  "  +   
files[i]);
           indexDocs(new File(file, files[i]),flag,dir,iwc);
         }
       }
    }
     else {
         boolean flags=true;
       try {
         fis = new FileInputStream(file);
      } catch (FileNotFoundException fnfe) {
        fnfe.printStackTrace();
      }
       try {
           Document doc = new Document();
           LineNumberReader lnr=new LineNumberReader(new FileReader(file));
           Field pathField = new StringField("path", file.getPath(), 
Field.Store.YES);
           doc.add(pathField);
           String line=null;
           System.out.println("INITIALIZING");
           int i=0;
           doc.add(new StringField("TT",file.getName(),Field.Store.YES));
           BufferedReader br=new BufferedReader(new 
InputStreamReader(fis));
            doc.add(new TextField("DD", br));
           while(flags)
           {
               System.out.println("Looping");
               IndexWriter iwcTemp1=new IndexWriter(dir,iwc);
           while( null != (line = lnr.readLine()) ){
                 i++;
             StringField sf=new 
StringField("EEE",line.trim(),Field.Store.YES);
                 doc.add(sf);
               if(i%10000==0)
               {
                   System.out.println("Breaking" +  i);
                   lnr.mark(i);
                   break;
               }
               sf=null;
           }
           if(line==null)
           {
               System.out.println("FALSE");
               flags=false;
           }
           System.out.println("Total value is  "  +  i);
           if (iwcTemp1.getConfig().getOpenMode() == 
OpenMode.CREATE_OR_APPEND) {
               try
               {
                 iwcTemp1.addDocument(doc);
                   iwcTemp1.commit();
                   iwcTemp1.close();
               }catch(Throwable t)
               {
                   lnr.close();
                       br.close();
                       fis.close();
                      t.printStackTrace();
               }

           } else {
               try
               {

             System.out.println("updating " + file);
             iwcTemp1.updateDocument(new Term("path", file.getPath()), doc);
               }catch(Exception e)
               {
                   e.printStackTrace();
               }
           }
           System.out.println("END OF WHILE  ");
           lnr.reset();
           }//end of While
       }catch (Exception e) {
          e.printStackTrace();
       }finally {
        fis.close();
       }
     }
   }
}
}


Kindly guide as to where the possible problem lies. Trying to figure out 
but to no avail..

-- 
Regards

Ankit Murarka

"What lies behind us and what lies before us are tiny matters compared with what lies within
us"


Mime
  • Unnamed multipart/alternative (inline, None, 0 bytes)
View raw message