james-server-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From btell...@apache.org
Subject [10/18] james-project git commit: MAILBOX-266 Set the size of SearchResponses to 1 in ElasticSearch tests
Date Wed, 06 Apr 2016 09:21:54 GMT
MAILBOX-266 Set the size of SearchResponses to 1 in ElasticSearch tests

This allows to ensure Scrolls are well performed


Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/b99d0c7f
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/b99d0c7f
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/b99d0c7f

Branch: refs/heads/master
Commit: b99d0c7f0164e431a4cbaa3e8e8fc19030ba3a6a
Parents: 6ba31a6
Author: Benoit Tellier <btellier@linagora.com>
Authored: Mon Mar 28 10:26:36 2016 +0700
Committer: Benoit Tellier <btellier@linagora.com>
Committed: Wed Apr 6 16:18:45 2016 +0700

----------------------------------------------------------------------
 .../mailbox/elasticsearch/DeleteByQueryPerformer.java  | 13 +++++++++++--
 .../elasticsearch/search/ElasticSearchSearcher.java    | 12 ++++++++++--
 .../elasticsearch/ElasticSearchIndexerTest.java        |  3 ++-
 .../elasticsearch/ElasticSearchIntegrationTest.java    |  6 ++++--
 4 files changed, 27 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java
index 9d266ce..895f016 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java
@@ -34,17 +34,26 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.SearchHit;
 
+import com.google.common.annotations.VisibleForTesting;
+
 public class DeleteByQueryPerformer {
-    public static final int BATCH_SIZE = 100;
+    public static final int DEFAULT_BATCH_SIZE = 100;
     public static final TimeValue TIMEOUT = new TimeValue(60000);
 
     private final ClientProvider clientProvider;
     private final ExecutorService executor;
+    private final int batchSize;
 
     @Inject
     public DeleteByQueryPerformer(ClientProvider clientProvider, @Named("AsyncExecutor")
ExecutorService executor) {
+        this(clientProvider, executor, DEFAULT_BATCH_SIZE);
+    }
+
+    @VisibleForTesting
+    DeleteByQueryPerformer(ClientProvider clientProvider, @Named("AsyncExecutor") ExecutorService
executor, int batchSize) {
         this.clientProvider = clientProvider;
         this.executor = executor;
+        this.batchSize = batchSize;
     }
 
     public void perform(QueryBuilder queryBuilder) {
@@ -59,7 +68,7 @@ public class DeleteByQueryPerformer {
                     .setScroll(TIMEOUT)
                     .setNoFields()
                     .setQuery(queryBuilder)
-                    .setSize(BATCH_SIZE));
+                    .setSize(batchSize));
             for (SearchResponse searchResponse : scrollIterable) {
                 deleteRetrievedIds(client, searchResponse);
             }

http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
index 96d90b5..ab4698e 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
@@ -46,14 +46,22 @@ import org.slf4j.LoggerFactory;
 public class ElasticSearchSearcher<Id extends MailboxId> {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchSearcher.class);
+    private static final TimeValue TIMEOUT = new TimeValue(60000);
+    public static final int DEFAULT_SIZE = 100;
 
     private final ClientProvider clientProvider;
     private final QueryConverter queryConverter;
+    private final int size;
 
     @Inject
     public ElasticSearchSearcher(ClientProvider clientProvider, QueryConverter queryConverter)
{
+        this(clientProvider, queryConverter, DEFAULT_SIZE);
+    }
+
+    public ElasticSearchSearcher(ClientProvider clientProvider, QueryConverter queryConverter,
int size) {
         this.clientProvider = clientProvider;
         this.queryConverter = queryConverter;
+        this.size = size;
     }
 
     public Iterator<Long> search(Mailbox<Id> mailbox, SearchQuery searchQuery)
throws MailboxException {
@@ -70,9 +78,9 @@ public class ElasticSearchSearcher<Id extends MailboxId> {
             .reduce(
                 client.prepareSearch(ElasticSearchIndexer.MAILBOX_INDEX)
                     .setTypes(ElasticSearchIndexer.MESSAGE_TYPE)
-                    .setScroll(new TimeValue(60000))
+                    .setScroll(TIMEOUT)
                     .setQuery(queryConverter.from(searchQuery, mailbox.getMailboxId().serialize()))
-                    .setSize(100),
+                    .setSize(size),
                 (searchBuilder, sort) -> searchBuilder.addSort(SortConverter.convertSort(sort)),
                 (partialResult1, partialResult2) -> partialResult1);
     }

http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java
b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java
index 1e4ce1a..7501c57 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java
@@ -41,6 +41,7 @@ import com.google.common.collect.Lists;
 
 public class ElasticSearchIndexerTest {
 
+    public static final int MINIMUM_BATCH_SIZE = 1;
     private TemporaryFolder temporaryFolder = new TemporaryFolder();
     private EmbeddedElasticSearch embeddedElasticSearch= new EmbeddedElasticSearch(temporaryFolder);
 
@@ -55,7 +56,7 @@ public class ElasticSearchIndexerTest {
     public void setup() throws IOException {
         node = embeddedElasticSearch.getNode();
         TestingClientProvider clientProvider = new TestingClientProvider(node);
-        deleteByQueryPerformer = new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor());
+        deleteByQueryPerformer = new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor(),
MINIMUM_BATCH_SIZE);
         testee = new ElasticSearchIndexer(clientProvider, deleteByQueryPerformer);
     }
     

http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
index 2f5962e..9b90b72 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
@@ -62,6 +62,8 @@ import com.google.common.collect.Lists;
 public class ElasticSearchIntegrationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchIntegrationTest.class);
+    public static final int BATCH_SIZE = 1;
+    public static final int SEARCH_SIZE = 1;
 
     private TemporaryFolder temporaryFolder = new TemporaryFolder();
     private EmbeddedElasticSearch embeddedElasticSearch= new EmbeddedElasticSearch(temporaryFolder);
@@ -168,8 +170,8 @@ public class ElasticSearchIntegrationTest {
         );
         MailboxSessionMapperFactory<InMemoryId> mapperFactory = new InMemoryMailboxSessionMapperFactory();
         elasticSearchListeningMessageSearchIndex = new ElasticSearchListeningMessageSearchIndex<>(mapperFactory,
-            new ElasticSearchIndexer(clientProvider, new DeleteByQueryPerformer(clientProvider,
Executors.newSingleThreadExecutor())),
-            new ElasticSearchSearcher<>(clientProvider, new QueryConverter(new CriterionConverter())),
+            new ElasticSearchIndexer(clientProvider, new DeleteByQueryPerformer(clientProvider,
Executors.newSingleThreadExecutor(), BATCH_SIZE)),
+            new ElasticSearchSearcher<>(clientProvider, new QueryConverter(new CriterionConverter()),
SEARCH_SIZE),
             new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.of("Europe/Paris")));
         storeMailboxManager = new StoreMailboxManager<>(
             mapperFactory,


---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org


Mime
View raw message