james-server-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From btell...@apache.org
Subject [11/12] james-project git commit: JAMES-2555 Proves our fix helps reindexing big mailboxes
Date Fri, 12 Oct 2018 08:29:02 GMT
JAMES-2555 Proves our fix helps reindexing big mailboxes

Before the fix the added test was failing with OutOfMemory exception after 300
messages. Now it passes easily 1000 messages.


Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/962683d3
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/962683d3
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/962683d3

Branch: refs/heads/master
Commit: 962683d38715e7a40286e422fc0487c495a47c32
Parents: 12a3445
Author: Benoit Tellier <btellier@linagora.com>
Authored: Thu Oct 11 09:19:44 2018 +0700
Committer: Benoit Tellier <btellier@linagora.com>
Committed: Fri Oct 12 15:27:34 2018 +0700

----------------------------------------------------------------------
 mailbox/tools/indexer/pom.xml                   |  53 ++++++++-
 .../indexer/CassandraReIndexerImplTest.java     | 119 +++++++++++++++++++
 2 files changed, 170 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/james-project/blob/962683d3/mailbox/tools/indexer/pom.xml
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/pom.xml b/mailbox/tools/indexer/pom.xml
index e0874f0..4395bba 100644
--- a/mailbox/tools/indexer/pom.xml
+++ b/mailbox/tools/indexer/pom.xml
@@ -34,6 +34,17 @@
     <dependencies>
         <dependency>
             <groupId>${james.groupId}</groupId>
+            <artifactId>apache-james-backends-cassandra</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${james.groupId}</groupId>
+            <artifactId>apache-james-backends-cassandra</artifactId>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${james.groupId}</groupId>
             <artifactId>apache-james-mailbox-api</artifactId>
         </dependency>
         <dependency>
@@ -44,11 +55,12 @@
         </dependency>
         <dependency>
             <groupId>${james.groupId}</groupId>
-            <artifactId>apache-james-mailbox-store</artifactId>
+            <artifactId>apache-james-mailbox-cassandra</artifactId>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>${james.groupId}</groupId>
-            <artifactId>apache-james-mailbox-store</artifactId>
+            <artifactId>apache-james-mailbox-cassandra</artifactId>
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
@@ -64,6 +76,26 @@
             <scope>test</scope>
         </dependency>
         <dependency>
+            <groupId>${james.groupId}</groupId>
+            <artifactId>apache-james-mailbox-store</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>${james.groupId}</groupId>
+            <artifactId>apache-james-mailbox-store</artifactId>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>${james.groupId}</groupId>
+            <artifactId>blob-cassandra</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
             <scope>test</scope>
@@ -91,7 +123,24 @@
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <argLine>-Xms128m -Xmx256m</argLine>
+                    <reuseForks>false</reuseForks>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
 
 </project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/james-project/blob/962683d3/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/CassandraReIndexerImplTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/CassandraReIndexerImplTest.java
b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/CassandraReIndexerImplTest.java
new file mode 100644
index 0000000..7af0646
--- /dev/null
+++ b/mailbox/tools/indexer/src/test/java/org/apache/mailbox/tools/indexer/CassandraReIndexerImplTest.java
@@ -0,0 +1,119 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+
+package org.apache.mailbox.tools.indexer;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+
+import org.apache.james.backends.cassandra.CassandraCluster;
+import org.apache.james.backends.cassandra.DockerCassandraRule;
+import org.apache.james.mailbox.MailboxSession;
+import org.apache.james.mailbox.MessageManager;
+import org.apache.james.mailbox.cassandra.CassandraMailboxManager;
+import org.apache.james.mailbox.cassandra.CassandraMailboxManagerProvider;
+import org.apache.james.mailbox.cassandra.mail.MailboxAggregateModule;
+import org.apache.james.mailbox.indexer.ReIndexer;
+import org.apache.james.mailbox.model.MailboxPath;
+import org.apache.james.mailbox.store.MailboxSessionMapperFactory;
+import org.apache.james.mailbox.store.mail.model.Mailbox;
+import org.apache.james.mailbox.store.mail.model.MailboxMessage;
+import org.apache.james.mailbox.store.search.ListeningMessageSearchIndex;
+import org.apache.james.util.concurrency.ConcurrentTestRunner;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import com.google.common.base.Strings;
+
+public class CassandraReIndexerImplTest {
+    private static final String USERNAME = "benwa@apache.org";
+    public static final MailboxPath INBOX = MailboxPath.forUser(USERNAME, "INBOX");
+    private CassandraMailboxManager mailboxManager;
+    private ListeningMessageSearchIndex messageSearchIndex;
+
+    private ReIndexer reIndexer;
+
+    @ClassRule
+    public static DockerCassandraRule cassandraServer = new DockerCassandraRule();
+
+    private static CassandraCluster cassandra;
+
+    @BeforeClass
+    public static void setUpClass() {
+        cassandra = CassandraCluster.create(MailboxAggregateModule.MODULE_WITH_QUOTA, cassandraServer.getHost());
+    }
+
+    @Before
+    public void setUp() {
+        mailboxManager = CassandraMailboxManagerProvider.provideMailboxManager(cassandra.getConf(),
cassandra.getTypesProvider());
+        MailboxSessionMapperFactory mailboxSessionMapperFactory = mailboxManager.getMapperFactory();
+        messageSearchIndex = mock(ListeningMessageSearchIndex.class);
+        reIndexer = new ReIndexerImpl(mailboxManager, messageSearchIndex, mailboxSessionMapperFactory);
+    }
+
+    @After
+    public void tearDown() {
+        cassandra.clearTables();
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+        cassandra.closeCluster();
+    }
+
+    @Test
+    public void reIndexShouldBeWellPerformed() throws Exception {
+        // Given a mailbox with 1000 messages * 150 KB
+        MailboxSession systemSession = mailboxManager.createSystemSession(USERNAME);
+        mailboxManager.createMailbox(INBOX, systemSession);
+
+        byte[] bigBody = (Strings.repeat("header: value\r\n", 10000) + "\r\nbody").getBytes(StandardCharsets.UTF_8);
+
+        int threadCount = 10;
+        int operationCount = 100;
+        MessageManager mailbox = mailboxManager.getMailbox(INBOX, systemSession);
+        ConcurrentTestRunner.builder()
+            .operation((a, b) -> mailbox
+                .appendMessage(
+                    MessageManager.AppendCommand.builder().build(bigBody),
+                    systemSession))
+            .threadCount(threadCount)
+            .operationCount(operationCount)
+            .runSuccessfullyWithin(Duration.ofMinutes(10));
+
+        // When We re-index
+        reIndexer.reIndex(INBOX);
+
+        // The indexer is called for each message
+        verify(messageSearchIndex).deleteAll(any(MailboxSession.class), any(Mailbox.class));
+        verify(messageSearchIndex, times(threadCount * operationCount))
+            .add(any(MailboxSession.class), any(Mailbox.class),any(MailboxMessage.class));
+        verifyNoMoreInteractions(messageSearchIndex);
+    }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org


Mime
View raw message