james-server-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From matth...@apache.org
Subject james-project git commit: JAMES-1811 use json for html to text conversion
Date Wed, 27 Jul 2016 09:00:00 GMT
Repository: james-project
Updated Branches:
  refs/heads/master 7c89479a4 -> fbc09db2f


JAMES-1811 use json for html to text conversion


Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/fbc09db2
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/fbc09db2
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/fbc09db2

Branch: refs/heads/master
Commit: fbc09db2f877d32b2f6d57e96ae06c96668f0842
Parents: 7c89479
Author: Matthieu Baechler <matthieu.baechler@linagora.com>
Authored: Tue Jul 26 16:25:39 2016 +0200
Committer: Matthieu Baechler <matthieu.baechler@linagora.com>
Committed: Wed Jul 27 10:59:35 2016 +0200

----------------------------------------------------------------------
 mailbox/memory/pom.xml                          |  5 ++
 .../mailbox/inmemory/JsoupTextExtractor.java    | 50 ++++++++++++++++++++
 server/container/guice/memory-guice/pom.xml     |  4 --
 .../modules/data/MemoryDataJmapModule.java      |  7 +--
 .../modules/mailbox/MemoryMailboxModule.java    |  4 +-
 .../integration/SetMessagesMethodTest.java      |  3 +-
 .../cucumber/GetMessagesMethodStepdefs.java     |  3 +-
 7 files changed, 65 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/james-project/blob/fbc09db2/mailbox/memory/pom.xml
----------------------------------------------------------------------
diff --git a/mailbox/memory/pom.xml b/mailbox/memory/pom.xml
index 0667a91..a164e25 100644
--- a/mailbox/memory/pom.xml
+++ b/mailbox/memory/pom.xml
@@ -81,6 +81,11 @@
             <scope>test</scope>
         </dependency>
         <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.9.2</version>
+        </dependency>
+        <dependency>
             <groupId>org.xenei</groupId>
             <artifactId>junit-contracts</artifactId>
             <scope>test</scope>

http://git-wip-us.apache.org/repos/asf/james-project/blob/fbc09db2/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/JsoupTextExtractor.java
----------------------------------------------------------------------
diff --git a/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/JsoupTextExtractor.java
b/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/JsoupTextExtractor.java
new file mode 100644
index 0000000..6f2de77
--- /dev/null
+++ b/mailbox/memory/src/main/java/org/apache/james/mailbox/inmemory/JsoupTextExtractor.java
@@ -0,0 +1,50 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one   *
+ * or more contributor license agreements.  See the NOTICE file *
+ * distributed with this work for additional information        *
+ * regarding copyright ownership.  The ASF licenses this file   *
+ * to you under the Apache License, Version 2.0 (the            *
+ * "License"); you may not use this file except in compliance   *
+ * with the License.  You may obtain a copy of the License at   *
+ *                                                              *
+ *   http://www.apache.org/licenses/LICENSE-2.0                 *
+ *                                                              *
+ * Unless required by applicable law or agreed to in writing,   *
+ * software distributed under the License is distributed on an  *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
+ * KIND, either express or implied.  See the License for the    *
+ * specific language governing permissions and limitations      *
+ * under the License.                                           *
+ ****************************************************************/
+
+package org.apache.james.mailbox.inmemory;
+
+import java.io.InputStream;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.james.mailbox.store.extractor.ParsedContent;
+import org.apache.james.mailbox.store.extractor.TextExtractor;
+import org.jsoup.Jsoup;
+
+import com.google.common.collect.Maps;
+
+
+public class JsoupTextExtractor implements TextExtractor {
+
+    @Override
+    public ParsedContent extractContent(InputStream inputStream, String contentType, String
fileName) throws Exception {
+        Map<String, List<String>> emptyMetadata = Maps.newHashMap();
+        if (contentType != null) {
+           if (contentType.equals("text/plain")) {
+            return new ParsedContent(IOUtils.toString(inputStream), emptyMetadata);
+           }
+           if (contentType.equals("text/html")) {
+               String text = Jsoup.parse(IOUtils.toString(inputStream)).text();
+               return new ParsedContent(text, emptyMetadata);
+           }
+        }
+        return new ParsedContent(null, emptyMetadata);
+    }
+}

http://git-wip-us.apache.org/repos/asf/james-project/blob/fbc09db2/server/container/guice/memory-guice/pom.xml
----------------------------------------------------------------------
diff --git a/server/container/guice/memory-guice/pom.xml b/server/container/guice/memory-guice/pom.xml
index 53124a1..09ba08a 100644
--- a/server/container/guice/memory-guice/pom.xml
+++ b/server/container/guice/memory-guice/pom.xml
@@ -203,10 +203,6 @@
                 </dependency>
                 <dependency>
                     <groupId>${project.groupId}</groupId>
-                    <artifactId>apache-james-mailbox-tika</artifactId>
-                </dependency>
-                <dependency>
-                    <groupId>${project.groupId}</groupId>
                     <artifactId>james-server-data-memory</artifactId>
                 </dependency>
                 <dependency>

http://git-wip-us.apache.org/repos/asf/james-project/blob/fbc09db2/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/data/MemoryDataJmapModule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/data/MemoryDataJmapModule.java
b/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/data/MemoryDataJmapModule.java
index 02924b1..7eef012 100644
--- a/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/data/MemoryDataJmapModule.java
+++ b/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/data/MemoryDataJmapModule.java
@@ -25,8 +25,9 @@ import org.apache.james.jmap.api.vacation.VacationRepository;
 import org.apache.james.jmap.memory.access.MemoryAccessTokenRepository;
 import org.apache.james.jmap.memory.vacation.MemoryNotificationRegistry;
 import org.apache.james.jmap.memory.vacation.MemoryVacationRepository;
+import org.apache.james.mailbox.inmemory.JsoupTextExtractor;
+import org.apache.james.mailbox.store.extractor.DefaultTextExtractor;
 import org.apache.james.mailbox.store.extractor.TextExtractor;
-import org.apache.james.mailbox.tika.extractor.TikaTextExtractor;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Scopes;
@@ -44,7 +45,7 @@ public class MemoryDataJmapModule extends AbstractModule {
         bind(MemoryNotificationRegistry.class).in(Scopes.SINGLETON);
         bind(NotificationRegistry.class).to(MemoryNotificationRegistry.class);
 
-        bind(TikaTextExtractor.class).in(Scopes.SINGLETON);
-        bind(TextExtractor.class).to(TikaTextExtractor.class);
+        bind(DefaultTextExtractor.class).in(Scopes.SINGLETON);
+        bind(TextExtractor.class).to(JsoupTextExtractor.class);
     }
 }

http://git-wip-us.apache.org/repos/asf/james-project/blob/fbc09db2/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java
b/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java
index e735349..4bc7a0a 100644
--- a/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java
+++ b/server/container/guice/memory-guice/src/main/java/org/apache/james/modules/mailbox/MemoryMailboxModule.java
@@ -33,6 +33,7 @@ import org.apache.james.mailbox.exception.MailboxException;
 import org.apache.james.mailbox.inmemory.InMemoryId;
 import org.apache.james.mailbox.inmemory.InMemoryMailboxManager;
 import org.apache.james.mailbox.inmemory.InMemoryMailboxSessionMapperFactory;
+import org.apache.james.mailbox.inmemory.JsoupTextExtractor;
 import org.apache.james.mailbox.inmemory.mail.InMemoryModSeqProvider;
 import org.apache.james.mailbox.inmemory.mail.InMemoryUidProvider;
 import org.apache.james.mailbox.model.MailboxId;
@@ -49,7 +50,6 @@ import org.apache.james.mailbox.store.mail.UidProvider;
 import org.apache.james.mailbox.store.search.MessageSearchIndex;
 import org.apache.james.mailbox.store.search.SimpleMessageSearchIndex;
 import org.apache.james.mailbox.store.user.SubscriptionMapperFactory;
-import org.apache.james.mailbox.tika.extractor.TikaTextExtractor;
 import org.apache.james.modules.Names;
 
 import com.google.inject.AbstractModule;
@@ -80,7 +80,7 @@ public class MemoryMailboxModule extends AbstractModule {
         bind(GroupMembershipResolver.class).to(SimpleGroupMembershipResolver.class);
 
         bind(MessageSearchIndex.class).to(SimpleMessageSearchIndex.class);
-        bind(TextExtractor.class).to(TikaTextExtractor.class);
+        bind(TextExtractor.class).to(JsoupTextExtractor.class);
 
         bind(InMemoryMailboxSessionMapperFactory.class).in(Scopes.SINGLETON);
         bind(InMemoryModSeqProvider.class).in(Scopes.SINGLETON);

http://git-wip-us.apache.org/repos/asf/james-project/blob/fbc09db2/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java
b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java
index af0dc11..2bd9003 100644
--- a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java
+++ b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/SetMessagesMethodTest.java
@@ -37,6 +37,7 @@ import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.collection.IsMapWithSize.aMapWithSize;
 import static org.hamcrest.collection.IsMapWithSize.anEmptyMap;
 
+
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.time.ZonedDateTime;
@@ -47,7 +48,7 @@ import java.util.concurrent.TimeUnit;
 
 import javax.mail.Flags;
 
-import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.IOUtils;
 import org.apache.james.GuiceJamesServer;
 import org.apache.james.jmap.JmapAuthentication;
 import org.apache.james.jmap.api.access.AccessToken;

http://git-wip-us.apache.org/repos/asf/james-project/blob/fbc09db2/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/GetMessagesMethodStepdefs.java
----------------------------------------------------------------------
diff --git a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/GetMessagesMethodStepdefs.java
b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/GetMessagesMethodStepdefs.java
index bdaaff1..e12a0f4 100644
--- a/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/GetMessagesMethodStepdefs.java
+++ b/server/protocols/jmap-integration-testing/jmap-integration-testing-common/src/test/java/org/apache/james/jmap/methods/integration/cucumber/GetMessagesMethodStepdefs.java
@@ -275,7 +275,8 @@ public class GetMessagesMethodStepdefs {
 
     @Then("^the preview of the message is \"([^\"]*)\"$")
     public void assertPreviewOfTheFirstMessage(String preview) throws Throwable {
-        assertThat(jsonPath.<String>read(FIRST_MESSAGE + ".preview")).isEqualTo(StringEscapeUtils.unescapeJava(preview));
+        String actual = jsonPath.<String>read(FIRST_MESSAGE + ".preview").replace("\n",
" ");
+        assertThat(actual).isEqualToIgnoringWhitespace(StringEscapeUtils.unescapeJava(preview));
     }
 
     @Then("^the headers of the message contains:$")


---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org


Mime
View raw message