james-server-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nor...@apache.org
Subject svn commit: r1130220 - in /james/mailbox/trunk: jcr/src/main/java/org/apache/james/mailbox/jcr/ jpa/src/main/java/org/apache/james/mailbox/jpa/ maildir/src/main/java/org/apache/james/mailbox/maildir/ maildir/src/main/java/org/apache/james/mailbox/maild...
Date Wed, 01 Jun 2011 16:49:05 GMT
Author: norman
Date: Wed Jun  1 16:49:05 2011
New Revision: 1130220

URL: http://svn.apache.org/viewvc?rev=1130220&view=rev
Log:
LuceneMessageSearchIndex now works... See MAILBOX-10

Added:
    james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java
    james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java
    james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java
      - copied, changed from r1130081, james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java
    james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java
Removed:
    james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java
Modified:
    james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java
    james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java
    james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java
    james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java
    james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java
    james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java
    james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java
    james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java
    james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java
    james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java

Modified: james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java
(original)
+++ james/mailbox/trunk/jcr/src/main/java/org/apache/james/mailbox/jcr/JCRMailboxSessionMapperFactory.java
Wed Jun  1 16:49:05 2011
@@ -29,8 +29,6 @@ import org.apache.james.mailbox.store.ma
 import org.apache.james.mailbox.store.mail.MessageMapper;
 import org.apache.james.mailbox.store.search.MessageSearchIndex;
 import org.apache.james.mailbox.store.user.SubscriptionMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * JCR implementation of a {@link MailboxSessionMapperFactory}
@@ -43,15 +41,14 @@ public class JCRMailboxSessionMapperFact
     private final static int DEFAULT_SCALING = 2;
     private final int scaling;
     private int messageScaling;
-    private final MessageSearchIndex<String> index;
 
     public JCRMailboxSessionMapperFactory(final MailboxSessionJCRRepository repository) {
         this(repository, null, DEFAULT_SCALING, JCRMessageMapper.MESSAGE_SCALE_DAY);
     }
 
     public JCRMailboxSessionMapperFactory(final MailboxSessionJCRRepository repository, final
MessageSearchIndex<String> index, final int scaling, final int messageScaling) {
+        super(index);
         this.repository = repository;
-        this.index = index;
         this.scaling = scaling;
         this.messageScaling = messageScaling;
     }
@@ -64,7 +61,7 @@ public class JCRMailboxSessionMapperFact
 
     @Override
     public MessageMapper<String> createMessageMapper(MailboxSession session) throws
MailboxException {
-        JCRMessageMapper messageMapper = new JCRMessageMapper(repository, session, index,
messageScaling);
+        JCRMessageMapper messageMapper = new JCRMessageMapper(repository, session, getSearchIndex(),
messageScaling);
         return messageMapper;
     }
 

Modified: james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java
(original)
+++ james/mailbox/trunk/jpa/src/main/java/org/apache/james/mailbox/jpa/JPAMailboxSessionMapperFactory.java
Wed Jun  1 16:49:05 2011
@@ -38,11 +38,10 @@ import org.apache.james.mailbox.store.us
 public class JPAMailboxSessionMapperFactory extends MailboxSessionMapperFactory<Long>
{
 
     private final EntityManagerFactory entityManagerFactory;
-    private MessageSearchIndex<Long> index;
 
     public JPAMailboxSessionMapperFactory(MessageSearchIndex<Long> index, EntityManagerFactory
entityManagerFactory) {
+        super(index);
         this.entityManagerFactory = entityManagerFactory;
-        this.index = index;
         createEntityManager().close();
     }
     
@@ -57,7 +56,7 @@ public class JPAMailboxSessionMapperFact
 
     @Override
     public MessageMapper<Long> createMessageMapper(MailboxSession session) {
-        return new JPAMessageMapper(session, index, entityManagerFactory);
+        return new JPAMessageMapper(session, getSearchIndex(), entityManagerFactory);
     }
 
     @Override

Modified: james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java
(original)
+++ james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/MaildirMailboxSessionMapperFactory.java
Wed Jun  1 16:49:05 2011
@@ -34,7 +34,6 @@ public class MaildirMailboxSessionMapper
         MailboxSessionMapperFactory<Integer> {
 
     private final MaildirStore store;
-    private final MessageSearchIndex<Integer> index;
     
     public MaildirMailboxSessionMapperFactory(MaildirStore store) {
         this(store, null);
@@ -42,8 +41,8 @@ public class MaildirMailboxSessionMapper
     
     
     public MaildirMailboxSessionMapperFactory(MaildirStore store, MessageSearchIndex<Integer>
index) {
+        super(index);
         this.store = store;
-        this.index = index;
     }
     
     
@@ -56,7 +55,7 @@ public class MaildirMailboxSessionMapper
     @Override
     protected MessageMapper<Integer> createMessageMapper(MailboxSession session)
             throws MailboxException {
-        return new MaildirMessageMapper(session, index, store);
+        return new MaildirMessageMapper(session, getSearchIndex(), store);
     }
 
     @Override

Modified: james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java
(original)
+++ james/mailbox/trunk/maildir/src/main/java/org/apache/james/mailbox/maildir/mail/model/MaildirMessage.java
Wed Jun  1 16:49:05 2011
@@ -28,7 +28,6 @@ import java.util.List;
 import javax.mail.Flags;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.NotImplementedException;
 import org.apache.james.mailbox.MailboxException;
 import org.apache.james.mailbox.store.mail.model.Header;
 import org.apache.james.mailbox.store.mail.model.Mailbox;
@@ -203,15 +202,6 @@ public class MaildirMessage extends Abst
         this.internalDate = date;
     }
 
-    /* 
-     * (non-Javadoc)
-     * @see org.apache.james.mailbox.store.mail.model.MailboxMembership#getMailboxId()
-     */
-    public Integer getMailboxId() {
-        throw new NotImplementedException();
-    }
-
-
     @Override
     public void setUid(long uid) {
         modified = true;

Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java
(original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/MailboxSessionMapperFactory.java
Wed Jun  1 16:49:05 2011
@@ -26,6 +26,8 @@ import org.apache.james.mailbox.store.ma
 import org.apache.james.mailbox.store.mail.MailboxMapperFactory;
 import org.apache.james.mailbox.store.mail.MessageMapper;
 import org.apache.james.mailbox.store.mail.MessageMapperFactory;
+import org.apache.james.mailbox.store.search.LazyMessageSearchIndex;
+import org.apache.james.mailbox.store.search.MessageSearchIndex;
 import org.apache.james.mailbox.store.transaction.Mapper;
 import org.apache.james.mailbox.store.user.SubscriptionMapper;
 import org.apache.james.mailbox.store.user.SubscriptionMapperFactory;
@@ -39,8 +41,24 @@ public abstract class MailboxSessionMapp
     protected final static String MESSAGEMAPPER ="MESSAGEMAPPER";
     protected final static String MAILBOXMAPPER ="MAILBOXMAPPER";
     protected final static String SUBSCRIPTIONMAPPER ="SUBSCRIPTIONMAPPER";
+    private MessageSearchIndex<Id> index;
 
+    public MailboxSessionMapperFactory(MessageSearchIndex<Id> index) {
+        this.index = index;
+        if (index != null && index instanceof LazyMessageSearchIndex<?>) {
+            ((LazyMessageSearchIndex<Id>) index).setMessageMapperFactory(this);
+        }
+    }
 
+    public MailboxSessionMapperFactory() {
+        this(null);
+    }
+    
+    protected MessageSearchIndex<Id> getSearchIndex() {
+        return index;
+    }
+    
+    
     /*
      * (non-Javadoc)
      * @see org.apache.james.mailbox.store.MessageMapperFactory#getMessageMapper(org.apache.james.mailbox.MailboxSession)

Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java
(original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/AbstractMessageMapper.java
Wed Jun  1 16:49:05 2011
@@ -252,12 +252,21 @@ public abstract class AbstractMessageMap
      * (non-Javadoc)
      * @see org.apache.james.mailbox.store.mail.MessageMapper#add(org.apache.james.mailbox.store.mail.model.Mailbox,
org.apache.james.mailbox.store.mail.model.Message)
      */
-    public MessageMetaData add(Mailbox<Id> mailbox, Message<Id> message) throws
MailboxException {
+    public MessageMetaData add(final Mailbox<Id> mailbox, Message<Id> message)
throws MailboxException {
         message.setUid(nextUid(mailbox));
         message.setModSeq(nextModSeq(mailbox));
         MessageMetaData data = save(mailbox, message);
         if (index != null) {
-            index.add(mailboxSession, mailbox, message);
+            findInMailbox(mailbox, MessageRange.one(data.getUid()), new MailboxMembershipCallback<Id>()
{
+
+                @Override
+                public void onMailboxMembers(List<Message<Id>> list) throws MailboxException
{
+                    if (!list.isEmpty()) {
+                        index.add(mailboxSession, mailbox, list.get(0));
+                    }
+                }
+            });
+            
         }
         return data;
         

Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java
(original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/mail/MessageMapper.java
Wed Jun  1 16:49:05 2011
@@ -25,7 +25,6 @@ import java.util.Map;
 import javax.mail.Flags;
 
 import org.apache.james.mailbox.MailboxException;
-import org.apache.james.mailbox.MailboxSession;
 import org.apache.james.mailbox.MessageMetaData;
 import org.apache.james.mailbox.MessageRange;
 import org.apache.james.mailbox.SearchQuery;

Added: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java?rev=1130220&view=auto
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java
(added)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/LazyMessageSearchIndex.java
Wed Jun  1 16:49:05 2011
@@ -0,0 +1,126 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+package org.apache.james.mailbox.store.search;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.mail.Flags;
+
+import org.apache.james.mailbox.MailboxException;
+import org.apache.james.mailbox.MailboxSession;
+import org.apache.james.mailbox.MessageRange;
+import org.apache.james.mailbox.SearchQuery;
+import org.apache.james.mailbox.store.mail.MessageMapperFactory;
+import org.apache.james.mailbox.store.mail.model.Mailbox;
+import org.apache.james.mailbox.store.mail.model.Message;
+import org.apache.james.mailbox.store.transaction.Mapper.MailboxMembershipCallback;
+
+/**
+ * {@link MessageSearchIndex} implementation which wraps another {@link MessageSearchIndex}
and will forward all calls to it.
+ * 
+ * The only special thing about this is that it will index all the mails in the mailbox on
the first call of {@link #search(MailboxSession, Mailbox, SearchQuery)}
+ * 
+ * This class is mostly useful for in-memory indexes or for indexed that should be recreated
on every server restart.
+ * 
+ *
+ * @param <Id>
+ */
+public class LazyMessageSearchIndex<Id> implements MessageSearchIndex<Id> {
+
+    private MessageSearchIndex<Id> index;
+    private MessageMapperFactory<Id> factory;
+    private final ConcurrentHashMap<Id, Boolean> indexed = new ConcurrentHashMap<Id,
Boolean>();
+    
+    
+    public LazyMessageSearchIndex(MessageSearchIndex<Id> index) {
+        this.index = index;
+    }
+    
+    
+    /**
+     * Set the {@link MessageMapperFactory} which will be used to access the {@link Message}'s
of the {@link Mailbox} on first {@link #search(MailboxSession, Mailbox, SearchQuery)}
+     * 
+     * This method must be called before the class can be used
+     * 
+     * @param factory
+     */
+    public void setMessageMapperFactory(MessageMapperFactory<Id> factory) {
+        this.factory = factory;
+    }
+    
+    
+    @Override
+    public void add(MailboxSession session, Mailbox<Id> mailbox, Message<Id>
message) throws MailboxException {    
+        index.add(session, mailbox, message);
+    }
+
+    @Override
+    public void delete(MailboxSession session, Mailbox<Id> mailbox, MessageRange range)
throws MailboxException {
+        index.delete(session, mailbox, range);
+    }
+
+    /**
+     * Lazy index the mailbox on first search request if it was not indexed before. After
indexing is done it delegate the search request to the wrapped
+     * {@link MessageSearchIndex}. Be aware that concurrent search requests are blocked on
the same "not-yet-indexed" mailbox till it the index process was 
+     * complete
+     * 
+     */
+    @Override
+    public Iterator<Long> search(final MailboxSession session, final Mailbox<Id>
mailbox, SearchQuery searchQuery) throws MailboxException {
+        Id id = mailbox.getMailboxId();
+        
+        Boolean done = indexed.get(id);
+        if (done == null) {
+            done = new Boolean(true);
+            Boolean oldDone = indexed.putIfAbsent(id, done);
+            if (oldDone != null) {
+                done = oldDone;
+            }
+            synchronized (done) {
+                factory.getMessageMapper(session).findInMailbox(mailbox, MessageRange.all(),
new MailboxMembershipCallback<Id>() {
+
+                    @Override
+                    public void onMailboxMembers(List<Message<Id>> list) throws
MailboxException {
+                        for (int i = 0; i < list.size(); i++) {
+                            final Message<Id> message = list.get(i);
+
+                            try {
+                                add(session, mailbox, message);
+                            } catch (MailboxException e) {
+                                session.getLog().debug("Unable to index message " + message.getUid()
+ " in mailbox " + mailbox.getName(), e);
+                            }
+
+                        }
+                    }
+                });
+            }
+        }
+       
+        return index.search(session, mailbox, searchQuery);
+    }
+
+
+    @Override
+    public void update(MailboxSession session, Mailbox<Id> mailbox, MessageRange range,
Flags flags) throws MailboxException {
+        index.update(session, mailbox, range, flags);
+    }
+
+}

Added: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java?rev=1130220&view=auto
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java
(added)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LenientImapSearchAnalyzer.java
Wed Jun  1 16:49:05 2011
@@ -0,0 +1,55 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+package org.apache.james.mailbox.store.search.lucene;
+
+import java.io.Reader;
+
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.shingle.ShingleFilter;
+import org.apache.lucene.util.Version;
+
+/**
+ * This {@link Analyzer} is not 100% conform with RFC3501 but does
+ * most times exactly what the user would expect. 
+ *
+ */
+public class LenientImapSearchAnalyzer extends Analyzer{
+
+    public final static int DEFAULT_MAX_TOKEN_LENGTH = 4;
+    
+    private final int maxTokenLength;
+    
+
+    public LenientImapSearchAnalyzer(int maxTokenLength) {
+        this.maxTokenLength = maxTokenLength;
+    }
+    
+    public LenientImapSearchAnalyzer() {
+        this(DEFAULT_MAX_TOKEN_LENGTH);
+    }
+    
+    @Override
+    public TokenStream tokenStream(String arg0, Reader reader) {
+        return new ShingleFilter(new LowerCaseFilter(Version.LUCENE_31, new WhitespaceTokenizer(Version.LUCENE_31,
reader)), 2, maxTokenLength);
+    }
+}

Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java
(original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/LuceneMessageSearchIndex.java
Wed Jun  1 16:49:05 2011
@@ -18,9 +18,11 @@
  ****************************************************************/
 package org.apache.james.mailbox.store.search.lucene;
 
-import java.io.ByteArrayOutputStream;
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
@@ -36,6 +38,7 @@ import org.apache.james.mailbox.MailboxE
 import org.apache.james.mailbox.MailboxSession;
 import org.apache.james.mailbox.MessageRange;
 import org.apache.james.mailbox.SearchQuery;
+import org.apache.james.mailbox.UnsupportedSearchException;
 import org.apache.james.mailbox.SearchQuery.AllCriterion;
 import org.apache.james.mailbox.SearchQuery.ContainsOperator;
 import org.apache.james.mailbox.SearchQuery.Criterion;
@@ -47,7 +50,6 @@ import org.apache.james.mailbox.SearchQu
 import org.apache.james.mailbox.SearchQuery.NumericOperator;
 import org.apache.james.mailbox.SearchQuery.NumericRange;
 import org.apache.james.mailbox.SearchQuery.UidCriterion;
-import org.apache.james.mailbox.UnsupportedSearchException;
 import org.apache.james.mailbox.store.mail.model.Mailbox;
 import org.apache.james.mailbox.store.mail.model.Message;
 import org.apache.james.mailbox.store.search.MessageSearchIndex;
@@ -60,14 +62,14 @@ import org.apache.james.mime4j.field.add
 import org.apache.james.mime4j.field.address.MailboxList;
 import org.apache.james.mime4j.message.Header;
 import org.apache.james.mime4j.message.SimpleContentHandler;
+import org.apache.james.mime4j.parser.MimeEntityConfig;
 import org.apache.james.mime4j.parser.MimeStreamParser;
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericField;
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.NumericField;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
@@ -84,12 +86,14 @@ import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.util.Version;
 
 /**
- * Lucene based {@link MessageSearchIndex} which offers message searching.
+ * Lucene based {@link MessageSearchIndex} which offers message searching via a Lucene index
+ * 
  * 
 
  * @param <Id>
@@ -128,6 +132,9 @@ public class LuceneMessageSearchIndex<Id
     public final static String BODY_FIELD = "body";
     
     
+    /**
+     * Prefix which will be used for each message header to store it also in a seperate {@link
Field}
+     */
     public final static String PREFIX_HEADER_FIELD ="header_";
     
     /**
@@ -135,22 +142,68 @@ public class LuceneMessageSearchIndex<Id
      */
     public final static String HEADERS_FIELD ="headers";
 
+    /**
+     * {@link Field} which will contain the mod-sequence of the message
+     */
     public final static String MODSEQ_FIELD = "modSeq";
 
-    
+    /**
+     * {@link Field} which will contain the TO-Address of the message
+     */
     public final static String TO_FIELD ="to";
+    
+
+    /**
+     * {@link Field} which will contain the CC-Address of the message
+     */
     public final static String CC_FIELD ="cc";
-    public final static String BCC_FIELD ="bcc";
-    public final static String FROM_FIELD ="from";
 
+    /**
+     * {@link Field} which will contain the BCC-Address of the message
+     */
+    public final static String BCC_FIELD ="bcc";
+    
 
+    /**
+     * {@link Field} which will contain the FROM-Address of the message
+     */
+    public final static String FROM_FIELD ="from";
+    
+    /**
+     * {@link Field} which contain the internalDate of the message with YEAR-Resolution
+     */
     public final static String INTERNAL_DATE_FIELD_YEAR_RESOLUTION ="internaldateYearResolution";
- 
+    
+    
+    /**
+     * {@link Field} which contain the internalDate of the message with MONTH-Resolution
+     */
     public final static String INTERNAL_DATE_FIELD_MONTH_RESOLUTION ="internaldateMonthResolution";
+    
+    /**
+     * {@link Field} which contain the internalDate of the message with DAY-Resolution
+     */
     public final static String INTERNAL_DATE_FIELD_DAY_RESOLUTION ="internaldateDayResolution";
+    
+    /**
+     * {@link Field} which contain the internalDate of the message with HOUR-Resolution
+     */
     public final static String INTERNAL_DATE_FIELD_HOUR_RESOLUTION ="internaldateHourResolution";
+    
+    /**
+     * {@link Field} which contain the internalDate of the message with MINUTE-Resolution
+     */
     public final static String INTERNAL_DATE_FIELD_MINUTE_RESOLUTION ="internaldateMinuteResolution";
+    
+    /**
+     * {@link Field} which contain the internalDate of the message with SECOND-Resolution
+     */
     public final static String INTERNAL_DATE_FIELD_SECOND_RESOLUTION ="internaldateSecondResolution";
+    
+    
+    /**
+     * {@link Field} which contain the internalDate of the message with MILLISECOND-Resolution
+     */
     public final static String INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION ="internaldateMillisecondResolution";
 
     /**
@@ -159,17 +212,26 @@ public class LuceneMessageSearchIndex<Id
     public final static String MAILBOX_ID_FIELD ="mailboxid";
 
     
+    
     private final static String MEDIA_TYPE_TEXT = "text"; 
     private final static String MEDIA_TYPE_MESSAGE = "message"; 
-
+    private final static String DEFAULT_ENCODING = "US-ASCII";
+    
     private final IndexWriter writer;
     
     private int maxQueryResults = DEFAULT_MAX_QUERY_RESULTS;
+
+    private boolean suffixMatch = false;
     
     private final static Sort UID_SORT = new Sort(new SortField(UID_FIELD, SortField.LONG));
     
-    public LuceneMessageSearchIndex(Directory directory, boolean breakIMAPRFC) throws CorruptIndexException,
LockObtainFailedException, IOException {
-        this(new IndexWriter(directory,  new IndexWriterConfig(Version.LUCENE_31, createAnalyzer(breakIMAPRFC))));
+    public LuceneMessageSearchIndex(Directory directory) throws CorruptIndexException, LockObtainFailedException,
IOException {
+        this(directory, true);
+    }
+    
+    
+    public LuceneMessageSearchIndex(Directory directory, boolean lenient) throws CorruptIndexException,
LockObtainFailedException, IOException {
+        this(new IndexWriter(directory,  new IndexWriterConfig(Version.LUCENE_31, createAnalyzer(lenient))));
     }
     
     
@@ -178,7 +240,7 @@ public class LuceneMessageSearchIndex<Id
     }
     
     /**
-     * Set the max count of results which will get returned from a query
+     * Set the max count of results which will get returned from a query. The default is
{@link #DEFAULT_MAX_QUERY_RESULTS}
      * 
      * @param maxQueryResults
      */
@@ -188,19 +250,33 @@ public class LuceneMessageSearchIndex<Id
     /**
      * Create a {@link Analyzer} which is used to index the {@link MailboxMembership}'s
      * 
-     * @param breakIMAPRFC 
+     * @param lenient 
      * 
      * @return analyzer
      */
-    private static Analyzer createAnalyzer(boolean breakIMAPRFC) {
-        if (breakIMAPRFC) {
-            return new SimpleAnalyzer(Version.LUCENE_31);
+    private static Analyzer createAnalyzer(boolean lenient) {
+        if (lenient) {
+           return new LenientImapSearchAnalyzer();
         } else {
-            return new ImapSearchAnalyzer();
+            return new StrictImapSearchAnalyzer();
         }
-
     }
     
+    /**
+     * If set to true this implementation will use {@link WildcardQuery} to match suffix
and prefix. This is what RFC3501 expects but is often not what the user does.
+     * It also slow things a lot if you have complex queries which use many "TEXT" arguments.
If you want the implementation to behave strict like RFC3501 says, you should
+     * set this to true. 
+     * 
+     * The default is false for performance reasons
+     * 
+     * 
+     * @param suffixMatch
+     */
+    public void setEnableSuffixMatch(boolean suffixMatch) {
+        this.suffixMatch = suffixMatch;
+    }
+    
+    
     
     /*
      * (non-Javadoc)
@@ -237,6 +313,7 @@ public class LuceneMessageSearchIndex<Id
         return uids.iterator();
     }
 
+   
     /**
      * Create a new {@link Document} for the given {@link MailboxMembership}. This Document
does not contain any flags data. The {@link Flags} are stored in a seperate Document. 
      * 
@@ -253,8 +330,6 @@ public class LuceneMessageSearchIndex<Id
         
         // create an unqiue key for the document which can be used later on updates to find
the document
         doc.add(new Field(ID_FIELD, membership.getMailboxId().toString().toLowerCase(Locale.US)
+"-" + Long.toString(membership.getUid()), Store.YES, Index.NOT_ANALYZED));
-        
-      
 
         doc.add(new NumericField(INTERNAL_DATE_FIELD_YEAR_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.YEAR).getTime()));
         doc.add(new NumericField(INTERNAL_DATE_FIELD_MONTH_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.MONTH).getTime()));
@@ -265,14 +340,11 @@ public class LuceneMessageSearchIndex<Id
         doc.add(new NumericField(INTERNAL_DATE_FIELD_MILLISECOND_RESOLUTION,Store.NO, true).setLongValue(DateUtils.truncate(membership.getInternalDate(),Calendar.MILLISECOND).getTime()));
 
         doc.add(new NumericField(SIZE_FIELD,Store.NO, true).setLongValue(membership.getFullContentOctets()));
-        
+
         // content handler which will index the headers and the body of the message
         SimpleContentHandler handler = new SimpleContentHandler() {
             
-            
-            /**
-             * Add the headers to the Document
-             */
+
             public void headers(Header header) {
                 
                 Iterator<org.apache.james.mime4j.parser.Field> fields = header.iterator();
@@ -321,32 +393,42 @@ public class LuceneMessageSearchIndex<Id
                 }
            
             }
-            
-
-
-            /**
-             * Add the body parts to the Document
+            /*
+             * (non-Javadoc)
+             * @see org.apache.james.mime4j.message.SimpleContentHandler#bodyDecoded(org.apache.james.mime4j.descriptor.BodyDescriptor,
java.io.InputStream)
              */
             public void bodyDecoded(BodyDescriptor desc, InputStream in) throws IOException
{
                 String mediaType = desc.getMediaType();
-                String charset = desc.getCharset();
                 if (MEDIA_TYPE_TEXT.equalsIgnoreCase(mediaType) || MEDIA_TYPE_MESSAGE.equalsIgnoreCase(mediaType))
{
-                    // TODO: maybe we want to limit the length here ?
-                    ByteArrayOutputStream out = new ByteArrayOutputStream();
-                    int b = -1;
-                    while ((b = in.read()) != -1) {
-                        out.write(b);
+                    String cset = desc.getCharset();
+                    if (cset == null) {
+                        cset = DEFAULT_ENCODING;
+                    }
+                    Charset charset;
+                    try {
+                        charset = Charset.forName(cset);
+                    } catch (Exception e) {
+                        // Invalid charset found so fallback toe the DEFAULT_ENCODING
+                        charset = Charset.forName(DEFAULT_ENCODING);
+                    }
+                    
+                    // Read the content one line after the other and add it to the document
+                    BufferedReader bodyReader = new BufferedReader(new InputStreamReader(in,
charset));
+                    String line = null;
+                    while((line = bodyReader.readLine()) != null) {
+                        doc.add(new Field(BODY_FIELD,  line.toLowerCase(Locale.US),Store.NO,
Index.ANALYZED));
                     }
-                    out.flush();
-                    doc.add(new Field(BODY_FIELD,  out.toString(charset).toLowerCase(Locale.US),Store.NO,
Index.ANALYZED));
-                    out.close();
                     
                 }
             }
         };
-        
-        MimeStreamParser parser = new MimeStreamParser();
+        MimeEntityConfig config = new MimeEntityConfig();
+        config.setMaxLineLen(-1);
+        config.setStrictParsing(false);
+        config.setMaxContentLen(-1);
+        MimeStreamParser parser = new MimeStreamParser(config);
         parser.setContentHandler(handler);
+       
         try {
             // parse the message to index headers and body
             parser.parse(membership.getFullContent());
@@ -358,7 +440,7 @@ public class LuceneMessageSearchIndex<Id
             // anyway let us just skip the body and headers in the index
             throw new MailboxException("Unable to index content of message", e);
         }
-
+       
 
         return doc;
     }
@@ -457,6 +539,20 @@ public class LuceneMessageSearchIndex<Id
     }
     
     /**
+     * This method will return the right {@link Query} depending if {@link #suffixMatch}
is enabled
+     * 
+     * @param fieldName
+     * @param value
+     * @return query
+     */
+    private Query createTermQuery(String fieldName, String value) {
+        if (suffixMatch) {
+            return new WildcardQuery(new Term(fieldName, "*" + value + "*"));
+        } else {
+            return new PrefixQuery(new Term(fieldName, value));
+        }
+    }
+    /**
      * Return a {@link Query} which is build based on the given {@link SearchQuery.HeaderCriterion}
      * 
      * @param crit
@@ -468,11 +564,11 @@ public class LuceneMessageSearchIndex<Id
         String fieldName = PREFIX_HEADER_FIELD + crit.getHeaderName().toLowerCase(Locale.US);
         if (op instanceof SearchQuery.ContainsOperator) {
             ContainsOperator cop = (ContainsOperator) op;
-            return new PrefixQuery(new Term(fieldName, cop.getValue().toLowerCase(Locale.US)));
+            return createTermQuery(fieldName, cop.getValue().toLowerCase(Locale.US));
         } else if (op instanceof SearchQuery.ExistsOperator){
             return new PrefixQuery(new Term(fieldName, ""));
         } else if (op instanceof SearchQuery.AddressOperator) {
-            return new PrefixQuery(new Term(fieldName.toLowerCase(), ((SearchQuery.AddressOperator)
op).getAddress().toLowerCase(Locale.US)));
+            return createTermQuery(fieldName.toLowerCase(), ((SearchQuery.AddressOperator)
op).getAddress().toLowerCase(Locale.US));
         } else {
             // Operator not supported
             throw new UnsupportedSearchException();
@@ -611,13 +707,14 @@ public class LuceneMessageSearchIndex<Id
      * @throws UnsupportedSearchException
      */
     private Query createTextQuery(SearchQuery.TextCriterion crit) throws UnsupportedSearchException
{
+        String value = crit.getOperator().getValue().toLowerCase(Locale.US);
         switch(crit.getType()) {
         case BODY:
-            return new PrefixQuery(new Term(BODY_FIELD, crit.getOperator().getValue().toLowerCase(Locale.US)));
+            return createTermQuery(BODY_FIELD, value);
         case FULL: 
             BooleanQuery query = new BooleanQuery();
-            query.add(new PrefixQuery(new Term(BODY_FIELD, crit.getOperator().getValue().toLowerCase(Locale.US))),
BooleanClause.Occur.SHOULD);
-            query.add(new PrefixQuery(new Term(HEADERS_FIELD, crit.getOperator().getValue().toLowerCase(Locale.US))),
BooleanClause.Occur.SHOULD);
+            query.add(createTermQuery(BODY_FIELD, value), BooleanClause.Occur.SHOULD);
+            query.add(createTermQuery(HEADERS_FIELD,value), BooleanClause.Occur.SHOULD);
             return query;
         default:
             throw new UnsupportedSearchException();

Copied: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java
(from r1130081, james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java)
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java?p2=james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java&p1=james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java&r1=1130081&r2=1130220&rev=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/ImapSearchAnalyzer.java
(original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/search/lucene/StrictImapSearchAnalyzer.java
Wed Jun  1 16:49:05 2011
@@ -38,15 +38,15 @@ import org.apache.lucene.util.Version;
 *      case-insensitive.
 *
 */
-public final class ImapSearchAnalyzer extends Analyzer {
+public final class StrictImapSearchAnalyzer extends Analyzer {
 
     private final int minTokenLength;
     private final int maxTokenLength;
     
-    public ImapSearchAnalyzer() {
+    public StrictImapSearchAnalyzer() {
         this(3, 40);
     }
-    public ImapSearchAnalyzer(int minTokenLength, int maxTokenLength) {
+    public StrictImapSearchAnalyzer(int minTokenLength, int maxTokenLength) {
         this.minTokenLength = minTokenLength;
         this.maxTokenLength = maxTokenLength;
     }

Modified: james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java
(original)
+++ james/mailbox/trunk/store/src/main/java/org/apache/james/mailbox/store/streaming/InputStreamContent.java
Wed Jun  1 16:49:05 2011
@@ -34,8 +34,7 @@ import org.apache.james.mailbox.store.ma
  *
  */
 public final class InputStreamContent implements org.apache.james.mailbox.InputStreamContent{
-    @SuppressWarnings("unchecked")
-    private Message m;
+    private Message<?> m;
     private Type type;
 
     public static enum Type {
@@ -43,8 +42,7 @@ public final class InputStreamContent im
         Body
     }
     
-    @SuppressWarnings("unchecked")
-    public InputStreamContent(Message m, Type type) throws IOException{
+    public InputStreamContent(Message<?> m, Type type) throws IOException{
         this.m = m;
         this.type = type;
     }

Modified: james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java?rev=1130220&r1=1130219&r2=1130220&view=diff
==============================================================================
--- james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java
(original)
+++ james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/LuceneMessageSearchIndexTest.java
Wed Jun  1 16:49:05 2011
@@ -40,7 +40,6 @@ import org.apache.james.mailbox.store.Si
 import org.apache.james.mailbox.store.SimpleMailboxMembership;
 import org.apache.james.mailbox.store.mail.model.Mailbox;
 import org.apache.james.mailbox.store.mail.model.Message;
-import org.apache.james.mailbox.store.search.MessageSearchIndex;
 import org.apache.james.mailbox.store.search.lucene.LuceneMessageSearchIndex;
 import org.apache.lucene.store.RAMDirectory;
 import org.junit.Before;
@@ -48,7 +47,7 @@ import org.junit.Test;
 
 public class LuceneMessageSearchIndexTest {
 
-    private MessageSearchIndex<Long> index;
+    private LuceneMessageSearchIndex<Long> index;
 
     private SimpleMailbox mailbox = new SimpleMailbox(0);
     private SimpleMailbox mailbox2 = new SimpleMailbox(1);
@@ -69,9 +68,14 @@ public class LuceneMessageSearchIndexTes
 
     Message<Long> row;
 
+    protected boolean useLenient() {
+        return true;
+    }
+    
     @Before
     public void setUp() throws Exception {
-        index = new LuceneMessageSearchIndex<Long>(new RAMDirectory(), false);
+        index = new LuceneMessageSearchIndex<Long>(new RAMDirectory(), useLenient());
+        index.setEnableSuffixMatch(true);
         List<org.apache.james.mailbox.store.SimpleHeader> headersSubject = new ArrayList<org.apache.james.mailbox.store.SimpleHeader>();
         headersSubject.add(new SimpleHeader("Subject", 1, "test"));
        

Added: james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java
URL: http://svn.apache.org/viewvc/james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java?rev=1130220&view=auto
==============================================================================
--- james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java
(added)
+++ james/mailbox/trunk/store/src/test/java/org/apache/james/mailbox/store/lucene/StrictLuceneMessageSearchIndexText.java
Wed Jun  1 16:49:05 2011
@@ -0,0 +1,28 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+package org.apache.james.mailbox.store.lucene;
+
+public class StrictLuceneMessageSearchIndexText extends LuceneMessageSearchIndexTest{
+
+    @Override
+    protected boolean useLenient() {
+        return false;
+    }
+
+}



---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org


Mime
View raw message