james-server-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From btell...@apache.org
Subject [1/4] james-project git commit: MAILBOX-330 Stop indexing attachment metadata
Date Wed, 18 Apr 2018 11:43:04 GMT
Repository: james-project
Updated Branches:
  refs/heads/master 3c379ddb4 -> e5fcb2ab5


MAILBOX-330 Stop indexing attachment metadata

No added value. Additionally metadata structure is guessed by ES, which
leads to some potential non sanitized dates failures.


Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/01dd7460
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/01dd7460
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/01dd7460

Branch: refs/heads/master
Commit: 01dd74605fae4c1a2ac88778812301f50693073f
Parents: 3c379dd
Author: benwa <btellier@linagora.com>
Authored: Wed Apr 18 11:28:48 2018 +0700
Committer: benwa <btellier@linagora.com>
Committed: Wed Apr 18 12:27:02 2018 +0700

----------------------------------------------------------------------
 .../json/JsonMessageConstants.java              |  1 -
 .../mailbox/elasticsearch/json/MimePart.java    | 21 +---------
 .../store/src/test/resources/eml/htmlMail.json  |  2 -
 .../src/test/resources/eml/nonTextual.json      | 43 --------------------
 .../src/test/resources/eml/recursiveMail.json   |  5 ---
 .../store/src/test/resources/eml/spamMail.json  |  3 --
 6 files changed, 2 insertions(+), 73 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
index d6fa74e..91e731b 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/JsonMessageConstants.java
@@ -72,7 +72,6 @@ public interface JsonMessageConstants {
         String CONTENT_DISPOSITION = "contentDisposition";
         String FILENAME = "fileName";
         String FILE_EXTENSION = "fileExtension";
-        String FILE_METADATA = "fileMetadata";
     }
 
     interface Property {

http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java
b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java
index 366cdd4..11f246c 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/json/MimePart.java
@@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
 
@@ -126,16 +125,7 @@ public class MimePart {
                 fileName,
                 fileExtension,
                 contentDisposition,
-                children,
-                parsedContent
-                    .map(x -> x.getMetadata()
-                        .entrySet()
-                        .stream()
-                        .reduce(ImmutableMultimap.<String, String>builder(),
-                            (builder, entry) -> builder.putAll(entry.getKey(), entry.getValue()),
-                            (builder1, builder2) -> builder1.putAll(builder2.build())).build())
-                    .orElse(ImmutableMultimap.of())
-            );
+                children);
         }
 
         private Optional<ParsedContent> parseContent(TextExtractor textExtractor) {
@@ -175,11 +165,10 @@ public class MimePart {
     private final Optional<String> fileExtension;
     private final Optional<String> contentDisposition;
     private final List<MimePart> attachments;
-    private final ImmutableMultimap<String, String> metadata;
 
     private MimePart(HeaderCollection headerCollection, Optional<String> bodyTextContent,
Optional<String> mediaType,
                     Optional<String> subType, Optional<String> fileName, Optional<String>
fileExtension,
-                    Optional<String> contentDisposition, List<MimePart> attachments,
Multimap<String, String> metadata) {
+                    Optional<String> contentDisposition, List<MimePart> attachments)
{
         this.headerCollection = headerCollection;
         this.mediaType = mediaType;
         this.subType = subType;
@@ -188,7 +177,6 @@ public class MimePart {
         this.contentDisposition = contentDisposition;
         this.attachments = attachments;
         this.bodyTextContent = bodyTextContent;
-        this.metadata = ImmutableMultimap.copyOf(metadata);
     }
 
     @JsonIgnore
@@ -236,11 +224,6 @@ public class MimePart {
         return bodyTextContent;
     }
 
-    @JsonProperty(JsonMessageConstants.Attachment.FILE_METADATA)
-    public ImmutableMultimap<String, String> getMetadata() {
-        return metadata;
-    }
-
     @JsonIgnore
     public Optional<String> locateFirstTextBody() {
         return firstBody(textAttachments()

http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/store/src/test/resources/eml/htmlMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/htmlMail.json b/mailbox/store/src/test/resources/eml/htmlMail.json
index 04cc4e3..30cbbf1 100644
--- a/mailbox/store/src/test/resources/eml/htmlMail.json
+++ b/mailbox/store/src/test/resources/eml/htmlMail.json
@@ -124,7 +124,6 @@
       "subtype":"plain",
       "fileName":null,
       "fileExtension":null,
-      "fileMetadata":{},
       "contentDisposition":null,
       "headers":{
         "content-transfer-encoding":[
@@ -140,7 +139,6 @@
       "mediaType":"text",
       "subtype":"html",
       "fileName":null,
-      "fileMetadata":{},
       "fileExtension":null,
       "contentDisposition":null,
       "headers":{

http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/store/src/test/resources/eml/nonTextual.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/nonTextual.json b/mailbox/store/src/test/resources/eml/nonTextual.json
index c9f3ed9..1170d5a 100644
--- a/mailbox/store/src/test/resources/eml/nonTextual.json
+++ b/mailbox/store/src/test/resources/eml/nonTextual.json
@@ -85,11 +85,6 @@
       "fileName":null,
       "fileExtension":null,
       "contentDisposition":null,
-      "fileMetadata":{
-        "X-Parsed-By":["org.apache.tika.parser.DefaultParser","org.apache.tika.parser.txt.TXTParser"],
-        "Content-Encoding":["windows-1252"],
-        "Content-Type":["text/plain; charset=windows-1252"]
-      },
       "headers":{
         "content-transfer-encoding":["7bit"],
         "content-type":["text/plain; charset=utf-8"]
@@ -102,44 +97,6 @@
       "fileName":"toto.odt",
       "fileExtension":"odt",
       "contentDisposition":"attachment",
-      "fileMetadata":{
-        "date":["2015-06-18T12:43:07.489893918"],
-        "meta:paragraph-count":["1"],
-        "meta:word-count":["7"],
-        "Table-Count":["0"],
-        "generator":["LibreOffice/4.4.3.2$Linux_X86_64 LibreOffice_project/40m0$Build-2"],
-        "Word-Count":["7"],
-        "dcterms:created":["2015-06-18T12:41:25.197399866"],
-        "dcterms:modified":["2015-06-18T12:43:07.489893918"],
-        "Last-Modified":["2015-06-18T12:43:07.489893918"],
-        "nbPara":["1"],
-        "Last-Save-Date":["2015-06-18T12:43:07.489893918"],
-        "meta:object-count":["0"],
-        "meta:character-count":["47"],
-        "Paragraph-Count":["1"],
-        "nbImg":["0"],
-        "meta:save-date":["2015-06-18T12:43:07.489893918"],
-        "modified":["2015-06-18T12:43:07.489893918"],
-        "Edit-Time":["PT1M42S"],
-        "meta:image-count":["0"],
-        "Image-Count":["0"],
-        "nbCharacter":["47"],
-        "nbObject":["0"],
-        "nbPage":["1"],
-        "Object-Count":["0"],
-        "nbWord":["7"],
-        "Content-Type":["application/vnd.oasis.opendocument.text"],
-        "X-Parsed-By":["org.apache.tika.parser.DefaultParser","org.apache.tika.parser.odf.OpenDocumentParser"],
-        "meta:creation-date":["2015-06-18T12:41:25.197399866"],
-        "meta:table-count":["0"],
-        "Creation-Date":["2015-06-18T12:41:25.197399866"],
-        "xmpTPg:NPages":["1"],
-        "Character Count":["47"],
-        "editing-cycles":["2"],
-        "Page-Count":["1"],
-        "nbTab":["0"],
-        "meta:page-count":["1"]
-      },
       "headers":{
         "content-transfer-encoding":["base64"],
         "content-disposition":["attachment; filename=\"toto.odt\""],

http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/store/src/test/resources/eml/recursiveMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/recursiveMail.json b/mailbox/store/src/test/resources/eml/recursiveMail.json
index ba1ca5c..7a57fc6 100644
--- a/mailbox/store/src/test/resources/eml/recursiveMail.json
+++ b/mailbox/store/src/test/resources/eml/recursiveMail.json
@@ -98,7 +98,6 @@
       "textContent": "Forward as attachment !\n\n\n",
       "mediaType": "text",
       "subtype": "plain",
-      "fileMetadata":{},
       "fileName": null,
       "fileExtension": null,
       "contentDisposition": null,
@@ -115,7 +114,6 @@
       "textContent": null,
       "mediaType": null,
       "subtype": null,
-      "fileMetadata":{},
       "fileName": null,
       "fileExtension": null,
       "contentDisposition": null,
@@ -136,7 +134,6 @@
       "mediaType": null,
       "subtype": null,
       "fileName": null,
-      "fileMetadata":{},
       "fileExtension": null,
       "contentDisposition": null,
       "headers": {
@@ -182,7 +179,6 @@
       "subtype": "plain",
       "fileName": null,
       "fileExtension": null,
-      "fileMetadata":{},
       "contentDisposition": null,
       "headers": {
         "content-transfer-encoding": [
@@ -195,7 +191,6 @@
     },
     {
       "textContent": null,
-      "fileMetadata":{},
       "mediaType": "application",
       "subtype": "vnd.oasis.opendocument.spreadsheet",
       "fileName": "Sprint-2015-05-18.ods",

http://git-wip-us.apache.org/repos/asf/james-project/blob/01dd7460/mailbox/store/src/test/resources/eml/spamMail.json
----------------------------------------------------------------------
diff --git a/mailbox/store/src/test/resources/eml/spamMail.json b/mailbox/store/src/test/resources/eml/spamMail.json
index d133cd7..e101669 100644
--- a/mailbox/store/src/test/resources/eml/spamMail.json
+++ b/mailbox/store/src/test/resources/eml/spamMail.json
@@ -130,7 +130,6 @@
 	  "mediaType": "text",
 	  "subtype": "plain",
 	  "fileName": null,
-	  "fileMetadata":{},
 	  "fileExtension": null,
 	  "contentDisposition": "inline",
 	  "headers": {
@@ -151,7 +150,6 @@
 	  "subtype": "rfc822-headers",
 	  "fileName": "header",
 	  "fileExtension": "",
-	  "fileMetadata":{},
 	  "contentDisposition": "inline",
 	  "headers": {
 		"content-disposition": [
@@ -173,7 +171,6 @@
 	  "mediaType": "text",
 	  "subtype": "plain",
 	  "fileName": null,
-	  "fileMetadata":{},
 	  "fileExtension": null,
 	  "contentDisposition": "inline",
 	  "headers": {


---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org


Mime
View raw message