james-server-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rouaz...@apache.org
Subject [08/22] james-project git commit: JAMES-1894 ElasticSearch should index MessageId
Date Wed, 11 Jan 2017 16:45:35 GMT
JAMES-1894 ElasticSearch should index MessageId


Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/7a893c49
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/7a893c49
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/7a893c49

Branch: refs/heads/master
Commit: 7a893c49ad49a978689afd94a1e5b0c5f612cd5f
Parents: c679556
Author: Benoit Tellier <btellier@linagora.com>
Authored: Thu Dec 22 11:53:22 2016 +0700
Committer: Benoit Tellier <btellier@linagora.com>
Committed: Mon Jan 9 21:58:33 2017 +0700

----------------------------------------------------------------------
 .../mailbox/elasticsearch/NodeMappingFactory.java |  6 +++++-
 .../elasticsearch/json/IndexableMessage.java      | 18 +++++++++++++-----
 .../elasticsearch/json/JsonMessageConstants.java  |  3 ++-
 .../elasticsearch/query/CriterionConverter.java   |  2 +-
 .../elasticsearch/query/SortConverter.java        |  4 ++--
 .../search/ElasticSearchSearcher.java             |  4 ++--
 .../MailboxMessageToElasticSearchJsonTest.java    |  6 +++---
 .../store/src/test/resources/eml/htmlMail.json    |  3 ++-
 mailbox/store/src/test/resources/eml/mail.json    |  3 ++-
 .../store/src/test/resources/eml/nonTextual.json  |  3 ++-
 .../src/test/resources/eml/pgpSignedMail.json     |  3 ++-
 .../src/test/resources/eml/recursiveMail.json     |  3 ++-
 .../eml/recursiveMailWithoutAttachments.json      |  3 ++-
 .../store/src/test/resources/eml/spamMail.json    |  3 ++-
 14 files changed, 42 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/NodeMappingFactory.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/NodeMappingFactory.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/NodeMappingFactory.java
index 7ee7eda..751452c 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/NodeMappingFactory.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/NodeMappingFactory.java
@@ -70,7 +70,11 @@ public class NodeMappingFactory {
 
                     .startObject(ElasticSearchIndexer.MESSAGE_TYPE)
                         .startObject(PROPERTIES)
-                            .startObject(JsonMessageConstants.ID)
+                            .startObject(JsonMessageConstants.MESSAGE_ID)
+                                .field(TYPE, STRING)
+                                .field(INDEX, NOT_ANALYZED)
+                            .endObject()
+                            .startObject(JsonMessageConstants.UID)
                                 .field(TYPE, LONG)
                             .endObject()
                             .startObject(JsonMessageConstants.MODSEQ)

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
index 0896140..dec8b8e 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/IndexableMessage.java
@@ -33,6 +33,7 @@ import org.apache.james.mailbox.MessageUid;
 import org.apache.james.mailbox.elasticsearch.IndexAttachments;
 import org.apache.james.mailbox.elasticsearch.query.DateResolutionFormater;
 import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.model.MessageId;
 import org.apache.james.mailbox.store.mail.model.MailboxMessage;
 import org.apache.james.mailbox.store.mail.model.Property;
 import org.apache.james.mime4j.MimeException;
@@ -89,7 +90,8 @@ public class IndexableMessage {
     }
 
     private void copyMessageFields(MailboxMessage message, ZoneId zoneId) {
-        this.id = message.getUid();
+        this.messageId = message.getMessageId();
+        this.uid = message.getUid();
         this.mailboxId = message.getMailboxId().serialize();
         this.modSeq = message.getModSeq();
         this.size = message.getFullContentOctets();
@@ -127,7 +129,8 @@ public class IndexableMessage {
             .collect(Collectors.joining(" "));
     }
 
-    private MessageUid id;
+    private MessageId messageId;
+    private MessageUid uid;
     private String mailboxId;
     private List<String> users;
     private long modSeq;
@@ -156,9 +159,14 @@ public class IndexableMessage {
     private Optional<String> bodyHtml;
     private String text;
 
-    @JsonProperty(JsonMessageConstants.ID)
-    public Long getId() {
-        return id.asLong();
+    @JsonProperty(JsonMessageConstants.MESSAGE_ID)
+    public String getId() {
+        return messageId.serialize();
+    }
+
+    @JsonProperty(JsonMessageConstants.UID)
+    public Long getUid() {
+        return uid.asLong();
     }
 
     @JsonProperty(JsonMessageConstants.MAILBOX_ID)

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
index de91b89..f931051 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
@@ -24,7 +24,8 @@ public interface JsonMessageConstants {
     /*
     Properties defined by JMAP
      */
-    String ID = "id";
+    String MESSAGE_ID = "messageId";
+    String UID = "uid";
     String MAILBOX_ID = "mailboxId";
     String USERS = "users";
     String IS_UNREAD = "isUnread";

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java
index 180bf99..3904a92 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/CriterionConverter.java
@@ -229,7 +229,7 @@ public class CriterionConverter {
     }
 
     private QueryBuilder uidRangeFilter(SearchQuery.UidRange numericRange) {
-        return rangeQuery(JsonMessageConstants.ID)
+        return rangeQuery(JsonMessageConstants.UID)
                 .lte(numericRange.getHighValue().asLong())
                 .gte(numericRange.getLowValue().asLong());
     }

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/SortConverter.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/SortConverter.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/SortConverter.java
index 2c80ecb..05bc3a8 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/SortConverter.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/query/SortConverter.java
@@ -57,7 +57,7 @@ public class SortConverter {
             case SentDate :
                 return SortBuilders.fieldSort(JsonMessageConstants.SENT_DATE);
             case Uid :
-                return SortBuilders.fieldSort(JsonMessageConstants.ID);
+                return SortBuilders.fieldSort(JsonMessageConstants.UID);
             case DisplayFrom:
                 return SortBuilders.fieldSort(JsonMessageConstants.FROM + PATH_SEPARATOR
+ JsonMessageConstants.EMailer.NAME + PATH_SEPARATOR + NodeMappingFactory.RAW)
                     .setNestedPath(JsonMessageConstants.FROM);
@@ -65,7 +65,7 @@ public class SortConverter {
                 return SortBuilders.fieldSort(JsonMessageConstants.TO + PATH_SEPARATOR +
JsonMessageConstants.EMailer.NAME + PATH_SEPARATOR + NodeMappingFactory.RAW)
                     .setNestedPath(JsonMessageConstants.TO);
             case Id:
-                return SortBuilders.fieldSort(JsonMessageConstants.ID);
+                return SortBuilders.fieldSort(JsonMessageConstants.MESSAGE_ID);
             default:
                 throw new RuntimeException("Sort is not implemented");
         }

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
index 3b2c012..1a5cd1e 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
@@ -85,7 +85,7 @@ public class ElasticSearchSearcher {
                 client.prepareSearch(ElasticSearchIndexer.MAILBOX_INDEX)
                     .setTypes(ElasticSearchIndexer.MESSAGE_TYPE)
                     .setScroll(TIMEOUT)
-                    .addFields(JsonMessageConstants.ID, JsonMessageConstants.MAILBOX_ID)
+                    .addFields(JsonMessageConstants.UID, JsonMessageConstants.MAILBOX_ID,
JsonMessageConstants.MESSAGE_ID)
                     .setQuery(queryConverter.from(users, query))
                     .setSize(size),
                 (searchBuilder, sort) -> searchBuilder.addSort(SortConverter.convertSort(sort)),
@@ -101,7 +101,7 @@ public class ElasticSearchSearcher {
 
     private Optional<Pair<MailboxId, MessageUid>> extractContentFromHit(SearchHit
hit) {
         SearchHitField mailboxId = hit.field(JsonMessageConstants.MAILBOX_ID);
-        SearchHitField uid = hit.field(JsonMessageConstants.ID);
+        SearchHitField uid = hit.field(JsonMessageConstants.UID);
         if (mailboxId != null && uid != null) {
             Number uidAsNumber = uid.getValue();
             return Optional.of(Pair.of(mailboxIdFactory.fromString(mailboxId.getValue()),
MessageUid.of(uidAsNumber.longValue())));

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java
b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java
index 4ee2e78..596d774 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/json/MailboxMessageToElasticSearchJsonTest.java
@@ -35,13 +35,13 @@ import javax.mail.util.SharedByteArrayInputStream;
 import org.apache.commons.io.IOUtils;
 import org.apache.james.mailbox.FlagsBuilder;
 import org.apache.james.mailbox.MailboxSession.User;
-import org.apache.james.mailbox.elasticsearch.IndexAttachments;
 import org.apache.james.mailbox.MessageUid;
+import org.apache.james.mailbox.elasticsearch.IndexAttachments;
 import org.apache.james.mailbox.mock.MockMailboxSession;
 import org.apache.james.mailbox.model.MessageId;
 import org.apache.james.mailbox.model.TestId;
+import org.apache.james.mailbox.model.TestMessageId;
 import org.apache.james.mailbox.store.extractor.DefaultTextExtractor;
-import org.apache.james.mailbox.store.mail.model.DefaultMessageId;
 import org.apache.james.mailbox.store.mail.model.MailboxMessage;
 import org.apache.james.mailbox.store.mail.model.impl.PropertyBuilder;
 import org.apache.james.mailbox.store.mail.model.impl.SimpleMailboxMessage;
@@ -58,7 +58,7 @@ public class MailboxMessageToElasticSearchJsonTest {
     public static final int SIZE = 25;
     public static final int BODY_START_OCTET = 100;
     public static final TestId MAILBOX_ID = TestId.of(18L);
-    public static final MessageId MESSAGE_ID = new DefaultMessageId();
+    public static final MessageId MESSAGE_ID = TestMessageId.of(184L);
     public static final long MOD_SEQ = 42L;
     public static final MessageUid UID = MessageUid.of(25);
     public static final Charset CHARSET = Charsets.UTF_8;

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/store/src/test/resources/eml/htmlMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/htmlMail.json b/mailbox/store/src/test/resources/eml/htmlMail.json
index 942ba1f..fdc1465 100644
--- a/mailbox/store/src/test/resources/eml/htmlMail.json
+++ b/mailbox/store/src/test/resources/eml/htmlMail.json
@@ -1,5 +1,6 @@
 {
-  "id":25,
+  "id":"184",
+  "uid":25,
   "mailboxId":"18",
   "modSeq":42,
   "size":25,

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/store/src/test/resources/eml/mail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/mail.json b/mailbox/store/src/test/resources/eml/mail.json
index ed0f5c0..c734e45 100644
--- a/mailbox/store/src/test/resources/eml/mail.json
+++ b/mailbox/store/src/test/resources/eml/mail.json
@@ -1,5 +1,6 @@
 {
- "id": 25,
+ "id":"184",
+ "uid":25,
  "mailboxId": "18",
  "modSeq": 42,
  "size": 25,

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/store/src/test/resources/eml/nonTextual.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/nonTextual.json b/mailbox/store/src/test/resources/eml/nonTextual.json
index 0cdffc0..f15f412 100644
--- a/mailbox/store/src/test/resources/eml/nonTextual.json
+++ b/mailbox/store/src/test/resources/eml/nonTextual.json
@@ -1,5 +1,6 @@
 {
-  "id": 25,
+  "id":"184",
+  "uid":25,
   "mailboxId":"18",
   "modSeq":42,
   "size":25,

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/store/src/test/resources/eml/pgpSignedMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/pgpSignedMail.json b/mailbox/store/src/test/resources/eml/pgpSignedMail.json
index e4b004f..91bdc42 100644
--- a/mailbox/store/src/test/resources/eml/pgpSignedMail.json
+++ b/mailbox/store/src/test/resources/eml/pgpSignedMail.json
@@ -1,5 +1,6 @@
 {
-  "id": 25,
+  "id":"184",
+  "uid":25,
   "mailboxId": "18",
   "modSeq": 42,
   "size": 25,

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/store/src/test/resources/eml/recursiveMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/recursiveMail.json b/mailbox/store/src/test/resources/eml/recursiveMail.json
index 2aa8208..4ec9942 100644
--- a/mailbox/store/src/test/resources/eml/recursiveMail.json
+++ b/mailbox/store/src/test/resources/eml/recursiveMail.json
@@ -1,5 +1,6 @@
 {
-  "id": 25,
+  "id":"184",
+  "uid":25,
   "mailboxId": "18",
   "modSeq": 42,
   "size": 25,

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
index c17741e..06935fd 100644
--- a/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
+++ b/mailbox/store/src/test/resources/eml/recursiveMailWithoutAttachments.json
@@ -1,5 +1,6 @@
 {
-  "id": 25,
+  "id":"184",
+  "uid":25,
   "mailboxId": "18",
   "modSeq": 42,
   "size": 25,

http://git-wip-us.apache.org/repos/asf/james-project/blob/7a893c49/mailbox/store/src/test/resources/eml/spamMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/spamMail.json b/mailbox/store/src/test/resources/eml/spamMail.json
index 6e150b0..d8ede1e 100644
--- a/mailbox/store/src/test/resources/eml/spamMail.json
+++ b/mailbox/store/src/test/resources/eml/spamMail.json
@@ -1,5 +1,6 @@
 {
-  "id": 25,
+  "id":"184",
+  "uid":25,
   "mailboxId": "18",
   "modSeq": 42,
   "size": 25,


---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org


Mime
View raw message