james-server-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From btell...@apache.org
Subject [16/17] james-project git commit: JAMES-2555 Allowing per-user re-indexing
Date Mon, 15 Oct 2018 06:28:39 GMT
JAMES-2555 Allowing per-user re-indexing


Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/a8731817
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/a8731817
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/a8731817

Branch: refs/heads/master
Commit: a87318174034b918115191dc90ddbe4a620952f3
Parents: 10a19a6
Author: Benoit Tellier <btellier@linagora.com>
Authored: Fri Oct 12 17:41:28 2018 +0700
Committer: Benoit Tellier <btellier@linagora.com>
Committed: Mon Oct 15 13:19:39 2018 +0700

----------------------------------------------------------------------
 .../apache/james/mailbox/indexer/ReIndexer.java |  3 +
 .../mailbox/tools/indexer/ReIndexerImpl.java    |  6 ++
 .../tools/indexer/ReIndexerPerformer.java       | 46 ++++++++--
 .../mailbox/tools/indexer/ThrowsReIndexer.java  |  6 ++
 .../tools/indexer/UserReindexingTask.java       | 88 ++++++++++++++++++++
 .../tools/indexer/ReIndexerImplTest.java        | 25 ++++++
 6 files changed, 167 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java
----------------------------------------------------------------------
diff --git a/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java b/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java
index 94304a7..5ed156e 100644
--- a/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java
+++ b/mailbox/api/src/main/java/org/apache/james/mailbox/indexer/ReIndexer.java
@@ -19,12 +19,15 @@
 
 package org.apache.james.mailbox.indexer;
 
+import org.apache.james.core.User;
 import org.apache.james.mailbox.exception.MailboxException;
 import org.apache.james.mailbox.model.MailboxPath;
 import org.apache.james.task.Task;
 
 public interface ReIndexer {
 
+    Task reIndex(User user) throws MailboxException;
+
     Task reIndex(MailboxPath path) throws MailboxException;
 
     Task reIndex() throws MailboxException;

http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java
b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java
index 0ef72c4..8fd3786 100644
--- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java
+++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerImpl.java
@@ -21,6 +21,7 @@ package org.apache.mailbox.tools.indexer;
 
 import javax.inject.Inject;
 
+import org.apache.james.core.User;
 import org.apache.james.mailbox.indexer.ReIndexer;
 import org.apache.james.mailbox.model.MailboxPath;
 import org.apache.james.task.Task;
@@ -53,4 +54,9 @@ public class ReIndexerImpl implements ReIndexer {
         return new FullReindexingTask(reIndexerPerformer);
     }
 
+    @Override
+    public Task reIndex(User user) {
+        return new UserReindexingTask(reIndexerPerformer, user);
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
index ce88882..fb5bf89 100644
--- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
+++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ReIndexerPerformer.java
@@ -21,15 +21,19 @@ package org.apache.mailbox.tools.indexer;
 
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Function;
 
 import javax.inject.Inject;
 
+import org.apache.james.core.User;
 import org.apache.james.mailbox.MailboxManager;
 import org.apache.james.mailbox.MailboxSession;
 import org.apache.james.mailbox.MessageUid;
 import org.apache.james.mailbox.exception.MailboxException;
+import org.apache.james.mailbox.model.MailboxMetaData;
 import org.apache.james.mailbox.model.MailboxPath;
 import org.apache.james.mailbox.model.MessageRange;
+import org.apache.james.mailbox.model.search.MailboxQuery;
 import org.apache.james.mailbox.store.MailboxSessionMapperFactory;
 import org.apache.james.mailbox.store.mail.MessageMapper;
 import org.apache.james.mailbox.store.mail.model.Mailbox;
@@ -43,9 +47,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.github.fge.lambdas.Throwing;
+import com.google.common.collect.ImmutableList;
 
 class ReIndexerPerformer {
-
     private static final Logger LOGGER = LoggerFactory.getLogger(ReIndexerPerformer.class);
 
     private static final int NO_LIMIT = 0;
@@ -73,18 +77,37 @@ class ReIndexerPerformer {
         MailboxSession mailboxSession = mailboxManager.createSystemSession("re-indexing");
         LOGGER.info("Starting a full reindex");
         List<MailboxPath> mailboxPaths = mailboxManager.list(mailboxSession);
-        GlobalRegistration globalRegistration = new GlobalRegistration();
-        mailboxManager.addGlobalListener(globalRegistration, mailboxSession);
+
         try {
-            return handleFullReindexingIterations(mailboxPaths, globalRegistration, reprocessingContext);
+            return reIndex(mailboxPaths, mailboxSession, reprocessingContext);
         } finally {
-            mailboxManager.removeGlobalListener(globalRegistration, mailboxSession);
             LOGGER.info("Full reindex finished");
         }
     }
 
-    private Task.Result handleFullReindexingIterations(List<MailboxPath> mailboxPaths,
GlobalRegistration globalRegistration,
-                                                       ReprocessingContext reprocessingContext)
{
+    Task.Result reIndex(User user, ReprocessingContext reprocessingContext) throws MailboxException
{
+        MailboxSession mailboxSession = mailboxManager.createSystemSession(user.asString());
+        LOGGER.info("Starting a reindex for user {}", user.asString());
+        List<MailboxPath> mailboxPaths = mailboxManager.search(MailboxQuery.privateMailboxesBuilder(mailboxSession)
+            .build(), mailboxSession)
+            .stream()
+            .map(MailboxMetaData::getPath)
+            .collect(ImmutableList.toImmutableList());
+
+        try {
+            return reIndex(mailboxPaths, mailboxSession, reprocessingContext);
+        } finally {
+            LOGGER.info("User {} reindex finished", user.asString());
+        }
+    }
+
+    private Task.Result reIndex(List<MailboxPath> mailboxPaths, MailboxSession mailboxSession,
ReprocessingContext reprocessingContext) throws MailboxException {
+        return wrapInGlobalRegistration(mailboxSession,
+            globalRegistration -> handleMultiMailboxesReindexingIterations(mailboxPaths,
globalRegistration, reprocessingContext));
+    }
+
+    private Task.Result handleMultiMailboxesReindexingIterations(List<MailboxPath>
mailboxPaths, GlobalRegistration globalRegistration,
+                                                                 ReprocessingContext reprocessingContext)
{
         return mailboxPaths.stream()
             .map(globalRegistration::getPathToIndex)
             .flatMap(OptionalUtils::toStream)
@@ -136,4 +159,13 @@ class ReIndexerPerformer {
             .findFirst();
     }
 
+    private <T> T wrapInGlobalRegistration(MailboxSession session, Function<GlobalRegistration,
T> function) throws MailboxException {
+        GlobalRegistration globalRegistration = new GlobalRegistration();
+        mailboxManager.addGlobalListener(globalRegistration, session);
+        try {
+            return function.apply(globalRegistration);
+        } finally {
+            mailboxManager.removeGlobalListener(globalRegistration, session);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java
b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java
index 6734795..f0d86ae 100644
--- a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java
+++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/ThrowsReIndexer.java
@@ -19,6 +19,7 @@
 
 package org.apache.mailbox.tools.indexer;
 
+import org.apache.james.core.User;
 import org.apache.james.mailbox.exception.MailboxException;
 import org.apache.james.mailbox.indexer.ReIndexer;
 import org.apache.james.mailbox.model.MailboxPath;
@@ -35,4 +36,9 @@ public class ThrowsReIndexer implements ReIndexer {
     public Task reIndex() throws MailboxException {
         throw new MailboxException("Not implemented");
     }
+
+    @Override
+    public Task reIndex(User user) throws MailboxException {
+        throw new MailboxException("Not implemented");
+    }
 }

http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/UserReindexingTask.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/UserReindexingTask.java
b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/UserReindexingTask.java
new file mode 100644
index 0000000..dab53e7
--- /dev/null
+++ b/mailbox/tools/indexer/src/main/java/org/apache/mailbox/tools/indexer/UserReindexingTask.java
@@ -0,0 +1,88 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+
+package org.apache.mailbox.tools.indexer;
+
+import java.util.Optional;
+
+import javax.inject.Inject;
+
+import org.apache.james.core.User;
+import org.apache.james.mailbox.exception.MailboxException;
+import org.apache.james.task.Task;
+import org.apache.james.task.TaskExecutionDetails;
+
+public class UserReindexingTask implements Task {
+
+    public static final String USER_RE_INDEXING = "userReIndexing";
+
+    public static class AdditionalInformation implements TaskExecutionDetails.AdditionalInformation
{
+        private final ReprocessingContext reprocessingContext;
+        private final User user;
+
+        AdditionalInformation(ReprocessingContext reprocessingContext, User user) {
+            this.reprocessingContext = reprocessingContext;
+            this.user = user;
+        }
+
+        public int getSuccessfullyReprocessMailCount() {
+            return reprocessingContext.successfullyReprocessedMailCount();
+        }
+
+        public int getFailedReprocessedMailCount() {
+            return reprocessingContext.failedReprocessingMailCount();
+        }
+
+        public String getUser() {
+            return user.asString();
+        }
+    }
+
+    private final ReIndexerPerformer reIndexerPerformer;
+    private final User user;
+    private final AdditionalInformation additionalInformation;
+    private final ReprocessingContext reprocessingContext;
+
+    @Inject
+    public UserReindexingTask(ReIndexerPerformer reIndexerPerformer, User user) {
+        this.reIndexerPerformer = reIndexerPerformer;
+        this.user = user;
+        this.reprocessingContext = new ReprocessingContext();
+        this.additionalInformation = new AdditionalInformation(reprocessingContext, user);
+    }
+
+    @Override
+    public Result run() {
+        try {
+            return reIndexerPerformer.reIndex(user, reprocessingContext);
+        } catch (MailboxException e) {
+            return Result.PARTIAL;
+        }
+    }
+
+    @Override
+    public String type() {
+        return USER_RE_INDEXING;
+    }
+
+    @Override
+    public Optional<TaskExecutionDetails.AdditionalInformation> details() {
+        return Optional.of(additionalInformation);
+    }
+}

http://git-wip-us.apache.org/repos/asf/james-project/blob/a8731817/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java
b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java
index 003601e..d1fed69 100644
--- a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java
+++ b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/ReIndexerImplTest.java
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 
+import org.apache.james.core.User;
 import org.apache.james.mailbox.MailboxSession;
 import org.apache.james.mailbox.MessageManager;
 import org.apache.james.mailbox.acl.SimpleGroupMembershipResolver;
@@ -108,4 +109,28 @@ public class ReIndexerImplTest {
         assertThat(messageCaptor.getValue()).matches(message -> message.getMailboxId().equals(mailboxId)
             && message.getUid().equals(createdMessage.getUid()));
     }
+
+    @Test
+    void userReIndexShouldBeWellPerformed() throws Exception {
+        MailboxSession systemSession = mailboxManager.createSystemSession(USERNAME);
+        MailboxId mailboxId = mailboxManager.createMailbox(INBOX, systemSession).get();
+        ComposedMessageId createdMessage = mailboxManager.getMailbox(INBOX, systemSession)
+            .appendMessage(
+                MessageManager.AppendCommand.builder().build("header: value\r\n\r\nbody"),
+                systemSession);
+
+        reIndexer.reIndex(User.fromUsername(USERNAME)).run();
+        ArgumentCaptor<MailboxMessage> messageCaptor = ArgumentCaptor.forClass(MailboxMessage.class);
+        ArgumentCaptor<Mailbox> mailboxCaptor1 = ArgumentCaptor.forClass(Mailbox.class);
+        ArgumentCaptor<Mailbox> mailboxCaptor2 = ArgumentCaptor.forClass(Mailbox.class);
+
+        verify(messageSearchIndex).deleteAll(any(MailboxSession.class), mailboxCaptor1.capture());
+        verify(messageSearchIndex).add(any(MailboxSession.class), mailboxCaptor2.capture(),
messageCaptor.capture());
+        verifyNoMoreInteractions(messageSearchIndex);
+
+        assertThat(mailboxCaptor1.getValue()).matches(mailbox -> mailbox.getMailboxId().equals(mailboxId));
+        assertThat(mailboxCaptor2.getValue()).matches(mailbox -> mailbox.getMailboxId().equals(mailboxId));
+        assertThat(messageCaptor.getValue()).matches(message -> message.getMailboxId().equals(mailboxId)
+            && message.getUid().equals(createdMessage.getUid()));
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org


Mime
View raw message