james-server-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From btell...@apache.org
Subject [14/17] james-project git commit: JAMES-2555 Simplify reIndexing: no more needed message event tracking
Date Mon, 15 Oct 2018 06:28:37 GMT
JAMES-2555 Simplify reIndexing: no more needed message event tracking

Because we reindex a list of UID for a given mailbox, if:
 - A new message arrive it will be handled by the normal indexing process
 - A concurrently deleted message will be filtered out as it can not be retrieved from storage
 - And we will read the latest applied flag value

Note that global event tracking is still needed for mailbox path renames.


Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/9ff8a260
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/9ff8a260
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/9ff8a260

Branch: refs/heads/master
Commit: 9ff8a260ea27993c46eb660c64a8881d9cf9f4a8
Parents: f354e3c
Author: Benoit Tellier <btellier@linagora.com>
Authored: Fri Oct 12 16:59:03 2018 +0700
Committer: Benoit Tellier <btellier@linagora.com>
Committed: Mon Oct 15 13:18:20 2018 +0700

----------------------------------------------------------------------
 .../tools/indexer/ReIndexerPerformer.java       |  33 +----
 .../indexer/events/ImpactingMessageEvent.java   | 144 -------------------
 .../registrations/MailboxRegistration.java      |  70 ---------
 .../registrations/MailboxRegistrationTest.java  | 107 --------------
 4 files changed, 2 insertions(+), 352 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
index c2d471d..ce88882 100644
--- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
+++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
@@ -38,15 +38,11 @@ import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex;
 import org.apache.james.task.Task;
 import org.apache.james.util.OptionalUtils;
 import org.apache.james.util.streams.Iterators;
-import org.apache.mailbox.tools.indexer.events.ImpactingEventType;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent;
 import org.apache.mailbox.tools.indexer.registrations.GlobalRegistration;
-import org.apache.mailbox.tools.indexer.registrations.MailboxRegistration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.github.fge.lambdas.Throwing;
-import com.google.common.collect.Lists;
 
 class ReIndexerPerformer {
 
@@ -105,34 +101,27 @@ class ReIndexerPerformer {
     }
 
     private Task.Result reIndex(MailboxPath path, MailboxSession mailboxSession, ReprocessingContext
reprocessingContext) throws MailboxException {
-        MailboxRegistration mailboxRegistration = new MailboxRegistration(path);
         LOGGER.info("Intend to reindex {}", path.asString());
         Mailbox mailbox = mailboxSessionMapperFactory.getMailboxMapper(mailboxSession).findMailboxByPath(path);
         messageSearchIndex.deleteAll(mailboxSession, mailbox);
-        mailboxManager.addListener(path, mailboxRegistration, mailboxSession);
         try {
             return Iterators.toStream(
                 mailboxSessionMapperFactory.getMessageMapper(mailboxSession)
                     .findInMailbox(mailbox, MessageRange.all(), MessageMapper.FetchType.Metadata,
NO_LIMIT))
                 .map(MailboxMessage::getUid)
-                .map(uid -> handleMessageReIndexing(mailboxSession, mailboxRegistration,
mailbox, uid))
+                .map(uid -> handleMessageReIndexing(mailboxSession, mailbox, uid))
                 .peek(reprocessingContext::updateAccordingToReprocessingResult)
                 .reduce(Task::combine)
                 .orElse(Task.Result.COMPLETED);
         } finally {
             LOGGER.info("Finish to reindex {}", path.asString());
-            mailboxManager.removeListener(path, mailboxRegistration, mailboxSession);
         }
     }
 
-    private Task.Result handleMessageReIndexing(MailboxSession mailboxSession, MailboxRegistration
mailboxRegistration, Mailbox mailbox, MessageUid uid) {
+    private Task.Result handleMessageReIndexing(MailboxSession mailboxSession, Mailbox mailbox,
MessageUid uid) {
         try {
-            Optional<ImpactingMessageEvent> impactingMessageEvent = findMostRelevant(mailboxRegistration.getImpactingEvents(uid));
-
             Optional.of(uid)
-                .filter(x -> !wasDeleted(impactingMessageEvent))
                 .flatMap(Throwing.function(mUid -> fullyReadMessage(mailboxSession, mailbox,
mUid)))
-                .map(message -> messageUpdateRegardingEvents(message, impactingMessageEvent))
                 .ifPresent(Throwing.consumer(message -> messageSearchIndex.add(mailboxSession,
mailbox, message)));
             return Task.Result.COMPLETED;
         } catch (Exception e) {
@@ -141,24 +130,6 @@ class ReIndexerPerformer {
         }
     }
 
-    private Optional<ImpactingMessageEvent> findMostRelevant(List<ImpactingMessageEvent>
messageEvents) {
-        for (ImpactingMessageEvent impactingMessageEvent : messageEvents) {
-            if (impactingMessageEvent.getType().equals(ImpactingEventType.Deletion)) {
-                return Optional.of(impactingMessageEvent);
-            }
-        }
-        return Lists.reverse(messageEvents).stream().findFirst();
-    }
-
-    private boolean wasDeleted(Optional<ImpactingMessageEvent> impactingMessageEvent)
{
-        return impactingMessageEvent.map(ImpactingMessageEvent::wasDeleted).orElse(false);
-    }
-
-    private MailboxMessage messageUpdateRegardingEvents(MailboxMessage message, Optional<ImpactingMessageEvent>
impactingMessageEvent) {
-        impactingMessageEvent.flatMap(ImpactingMessageEvent::newFlags).ifPresent(message::setFlags);
-        return message;
-    }
-
     private Optional<MailboxMessage> fullyReadMessage(MailboxSession mailboxSession,
Mailbox mailbox, MessageUid mUid) throws MailboxException {
         return Iterators.toStream(mailboxSessionMapperFactory.getMessageMapper(mailboxSession)
             .findInMailbox(mailbox, MessageRange.one(mUid), MessageMapper.FetchType.Full,
SINGLE_MESSAGE))

http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java
b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java
deleted file mode 100644
index 527f463..0000000
--- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/events/ImpactingMessageEvent.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one   *
- * or more contributor license agreements.  See the NOTICE file *
- * distributed with this work for additional information        *
- * regarding copyright ownership.  The ASF licenses this file   *
- * to you under the Apache License, Version 2.0 (the            *
- * "License"); you may not use this file except in compliance   *
- * with the License.  You may obtain a copy of the License at   *
- *                                                              *
- *   http://www.apache.org/licenses/LICENSE-2.0                 *
- *                                                              *
- * Unless required by applicable law or agreed to in writing,   *
- * software distributed under the License is distributed on an  *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
- * KIND, either express or implied.  See the License for the    *
- * specific language governing permissions and limitations      *
- * under the License.                                           *
- ****************************************************************/
-
-package org.apache.mailbox.tools.indexer.events;
-
-import java.util.Objects;
-import java.util.Optional;
-
-import javax.mail.Flags;
-
-import org.apache.james.mailbox.MessageUid;
-import org.apache.james.mailbox.model.MailboxPath;
-
-public interface ImpactingMessageEvent extends ImpactingEvent {
-
-    MessageUid getUid();
-
-    Optional<Flags> newFlags();
-
-    boolean wasDeleted();
-
-    class FlagsMessageEvent implements ImpactingMessageEvent {
-
-        private final MailboxPath mailboxPath;
-        private final MessageUid uid;
-        private final Flags flags;
-
-        public FlagsMessageEvent(MailboxPath mailboxPath, MessageUid uid, Flags flags) {
-            this.mailboxPath = mailboxPath;
-            this.uid = uid;
-            this.flags = flags;
-        }
-
-        @Override
-        public MessageUid getUid() {
-            return uid;
-        }
-
-        @Override
-        public MailboxPath getMailboxPath() {
-            return mailboxPath;
-        }
-
-        @Override
-        public ImpactingEventType getType() {
-            return ImpactingEventType.FlagsUpdate;
-        }
-
-        @Override
-        public Optional<Flags> newFlags() {
-            return Optional.of(flags);
-        }
-
-        @Override
-        public boolean wasDeleted() {
-            return false;
-        }
-
-        @Override
-        public final boolean equals(Object o) {
-            if (o instanceof FlagsMessageEvent) {
-                FlagsMessageEvent that = (FlagsMessageEvent) o;
-
-                return Objects.equals(this.mailboxPath, that.mailboxPath)
-                    && Objects.equals(this.uid, that.uid)
-                    && Objects.equals(this.flags, that.flags);
-            }
-            return false;
-        }
-
-        @Override
-        public final int hashCode() {
-            return Objects.hash(mailboxPath, uid, flags);
-        }
-    }
-
-    class MessageDeletedEvent implements ImpactingMessageEvent {
-
-        private final MailboxPath mailboxPath;
-        private final MessageUid uid;
-
-        public MessageDeletedEvent(MailboxPath mailboxPath, MessageUid uid) {
-            this.mailboxPath = mailboxPath;
-            this.uid = uid;
-        }
-
-        @Override
-        public MessageUid getUid() {
-            return uid;
-        }
-
-        @Override
-        public MailboxPath getMailboxPath() {
-            return mailboxPath;
-        }
-
-        @Override
-        public ImpactingEventType getType() {
-            return ImpactingEventType.Deletion;
-        }
-
-        @Override
-        public Optional<Flags> newFlags() {
-            return Optional.empty();
-        }
-
-        @Override
-        public boolean wasDeleted() {
-            return true;
-        }
-
-        @Override
-        public final boolean equals(Object o) {
-            if (o instanceof MessageDeletedEvent) {
-                MessageDeletedEvent that = (MessageDeletedEvent) o;
-
-                return Objects.equals(this.mailboxPath, that.mailboxPath)
-                    && Objects.equals(this.uid, that.uid);
-            }
-            return false;
-        }
-
-        @Override
-        public final int hashCode() {
-            return Objects.hash(mailboxPath, uid);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java
b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java
deleted file mode 100644
index 251fde8..0000000
--- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistration.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one   *
- * or more contributor license agreements.  See the NOTICE file *
- * distributed with this work for additional information        *
- * regarding copyright ownership.  The ASF licenses this file   *
- * to you under the Apache License, Version 2.0 (the            *
- * "License"); you may not use this file except in compliance   *
- * with the License.  You may obtain a copy of the License at   *
- *                                                              *
- *   http://www.apache.org/licenses/LICENSE-2.0                 *
- *                                                              *
- * Unless required by applicable law or agreed to in writing,   *
- * software distributed under the License is distributed on an  *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
- * KIND, either express or implied.  See the License for the    *
- * specific language governing permissions and limitations      *
- * under the License.                                           *
- ****************************************************************/
-
-package org.apache.mailbox.tools.indexer.registrations;
-
-import java.util.List;
-
-import org.apache.james.mailbox.Event;
-import org.apache.james.mailbox.MailboxListener;
-import org.apache.james.mailbox.MessageUid;
-import org.apache.james.mailbox.model.MailboxPath;
-import org.apache.james.mailbox.model.UpdatedFlags;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.FlagsMessageEvent;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.MessageDeletedEvent;
-
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
-
-public class MailboxRegistration implements MailboxListener {
-
-    private final Multimap<MessageUid, ImpactingMessageEvent> impactingMessageEvents;
-    private final MailboxPath mailboxPath;
-
-    public MailboxRegistration(MailboxPath mailboxPath) {
-        this.impactingMessageEvents = Multimaps.synchronizedMultimap(ArrayListMultimap.<MessageUid,
ImpactingMessageEvent>create());
-        this.mailboxPath = mailboxPath;
-    }
-
-    @Override
-    public ListenerType getType() {
-        return ListenerType.MAILBOX;
-    }
-
-    public List<ImpactingMessageEvent> getImpactingEvents(MessageUid uid) {
-        return ImmutableList.copyOf(impactingMessageEvents.get(uid));
-    }
-
-    @Override
-    public void event(Event event) {
-        if (event instanceof FlagsUpdated) {
-            for (UpdatedFlags updatedFlags : ((FlagsUpdated) event).getUpdatedFlags()) {
-                impactingMessageEvents.put(updatedFlags.getUid(), new FlagsMessageEvent(mailboxPath,
updatedFlags.getUid(), updatedFlags.getNewFlags()));
-            }
-        } else if (event instanceof Expunged) {
-            for (MessageUid uid: ((Expunged) event).getUids()) {
-                impactingMessageEvents.put(uid, new MessageDeletedEvent(mailboxPath, uid));
-            }
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/james-project/blob/9ff8a260/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java
b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java
deleted file mode 100644
index 3b9bad2..0000000
--- a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/registrations/MailboxRegistrationTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one   *
- * or more contributor license agreements.  See the NOTICE file *
- * distributed with this work for additional information        *
- * regarding copyright ownership.  The ASF licenses this file   *
- * to you under the Apache License, Version 2.0 (the            *
- * "License"); you may not use this file except in compliance   *
- * with the License.  You may obtain a copy of the License at   *
- *                                                              *
- *   http://www.apache.org/licenses/LICENSE-2.0                 *
- *                                                              *
- * Unless required by applicable law or agreed to in writing,   *
- * software distributed under the License is distributed on an  *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
- * KIND, either express or implied.  See the License for the    *
- * specific language governing permissions and limitations      *
- * under the License.                                           *
- ****************************************************************/
-
-package org.apache.mailbox.tools.indexer.registrations;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import java.util.Date;
-import java.util.TreeMap;
-
-import javax.mail.Flags;
-
-import org.apache.james.mailbox.MailboxListener;
-import org.apache.james.mailbox.MessageUid;
-import org.apache.james.mailbox.mock.MockMailboxSession;
-import org.apache.james.mailbox.model.MailboxPath;
-import org.apache.james.mailbox.model.MessageMetaData;
-import org.apache.james.mailbox.model.UpdatedFlags;
-import org.apache.james.mailbox.store.SimpleMessageMetaData;
-import org.apache.james.mailbox.store.event.EventFactory;
-import org.apache.james.mailbox.store.mail.model.DefaultMessageId;
-import org.apache.james.mailbox.store.mail.model.MailboxMessage;
-import org.apache.james.mailbox.store.mail.model.impl.SimpleMailbox;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.FlagsMessageEvent;
-import org.apache.mailbox.tools.indexer.events.ImpactingMessageEvent.MessageDeletedEvent;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-
-public class MailboxRegistrationTest {
-
-    public static final MailboxPath INBOX = MailboxPath.forUser("btellier@apache.org", "INBOX");
-    private static final MessageUid UID = MessageUid.of(18);
-    private static final int UID_VALIDITY = 45;
-    private static final SimpleMailbox MAILBOX = new SimpleMailbox(INBOX, UID_VALIDITY);
-    private static final int MOD_SEQ = 21;
-    private static final int SIZE = 41;
-    private static final Flags NEW_FLAGS = new Flags(Flags.Flag.ANSWERED);
-    private MailboxRegistration mailboxRegistration;
-    private EventFactory eventFactory;
-    private MockMailboxSession session;
-
-    @BeforeEach
-    void setUp() {
-        session = new MockMailboxSession("test");
-        eventFactory = new EventFactory();
-        mailboxRegistration = new MailboxRegistration(INBOX);
-    }
-
-    @Test
-    void reportedEventsShouldBeInitiallyEmpty() {
-        assertThat(mailboxRegistration.getImpactingEvents(UID)).isEmpty();
-    }
-
-
-    @Test
-    void addedEventsShouldNotBeReported() {
-        TreeMap<MessageUid, MessageMetaData> treeMap = new TreeMap<>();
-        treeMap.put(UID, new SimpleMessageMetaData(UID, MOD_SEQ, new Flags(), SIZE, new Date(),
new DefaultMessageId()));
-        MailboxListener.MailboxEvent event = eventFactory.added(session, treeMap, MAILBOX,
ImmutableMap.<MessageUid, MailboxMessage>of());
-        mailboxRegistration.event(event);
-        assertThat(mailboxRegistration.getImpactingEvents(UID)).isEmpty();
-    }
-
-    @Test
-    void expungedEventsShouldBeReported() {
-        TreeMap<MessageUid, MessageMetaData> treeMap = new TreeMap<>();
-        treeMap.put(UID, new SimpleMessageMetaData(UID, MOD_SEQ, new Flags(), SIZE, new Date(),
new DefaultMessageId()));
-        MailboxListener.MailboxEvent event = eventFactory.expunged(session, treeMap, MAILBOX);
-        mailboxRegistration.event(event);
-        assertThat(mailboxRegistration.getImpactingEvents(UID)).containsExactly(new MessageDeletedEvent(INBOX,
UID));
-    }
-
-    @Test
-    void flagsEventsShouldBeReported() {
-        MailboxListener.MailboxEvent event = eventFactory.flagsUpdated(session,
-            Lists.newArrayList(UID),
-            MAILBOX,
-            Lists.newArrayList(UpdatedFlags.builder()
-                .uid(UID)
-                .modSeq(MOD_SEQ)
-                .oldFlags(new Flags())
-                .newFlags(NEW_FLAGS)
-                .build()));
-        mailboxRegistration.event(event);
-        assertThat(mailboxRegistration.getImpactingEvents(UID)).containsExactly(new FlagsMessageEvent(INBOX,
UID, NEW_FLAGS));
-    }
-
-}


---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org


Mime
View raw message