any23-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ans...@apache.org
Subject [4/6] any23 git commit: Ref ANY23-308
Date Thu, 13 Jul 2017 05:51:24 GMT
Ref ANY23-308

- restore csvutils
- detect yaml based on the file name
- remove utils module

Signed-off-by:Jacek Grzebyta <grzebyta.dev@gmail.com>

Project: http://git-wip-us.apache.org/repos/asf/any23/repo
Commit: http://git-wip-us.apache.org/repos/asf/any23/commit/9839e212
Tree: http://git-wip-us.apache.org/repos/asf/any23/tree/9839e212
Diff: http://git-wip-us.apache.org/repos/asf/any23/diff/9839e212

Branch: refs/heads/master
Commit: 9839e212ec265237021219c0fff120929906e7bf
Parents: 0b82fdc
Author: Jacek Grzebyta <grzebyta.dev@gmail.com>
Authored: Wed Jul 12 18:52:43 2017 +0100
Committer: Jacek Grzebyta <grzebyta.dev@gmail.com>
Committed: Wed Jul 12 18:52:43 2017 +0100

----------------------------------------------------------------------
 cli/src/test/resources/log4j.properties         |   2 +
 .../any23/extractor/yaml/YAMLExtractorTest.java |  13 --
 .../extractor/yaml/YAMLTikaParserTest.java      |  30 +++-
 csvutils/pom.xml                                | 106 ++++++++++++
 .../any23/extractor/csv/CSVReaderBuilder.java   | 166 +++++++++++++++++++
 csvutils/src/test/resources/log4j.properties    |  34 ++++
 mime/pom.xml                                    |   2 +-
 .../apache/any23/mime/TikaMIMETypeDetector.java |  14 --
 pom.xml                                         |   2 +-
 utils/pom.xml                                   | 123 --------------
 .../any23/extractor/csv/CSVReaderBuilder.java   | 166 -------------------
 .../any23/extractor/yaml/YAMLValidator.java     | 105 ------------
 .../any23/yaml/utils/YAMLValidatorTest.java     |  66 --------
 utils/src/test/resources/log4j.properties       |  35 ----
 14 files changed, 336 insertions(+), 528 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/cli/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/cli/src/test/resources/log4j.properties b/cli/src/test/resources/log4j.properties
index 1918a4b..f286f6a 100644
--- a/cli/src/test/resources/log4j.properties
+++ b/cli/src/test/resources/log4j.properties
@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+log4j.logger.org.apache.any23.cli.YAMLRoverTest=trace
+
 log4j.rootLogger=INFO, A1
 
 # A1 is set to be a ConsoleAppender.

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/core/src/test/java/org/apache/any23/extractor/yaml/YAMLExtractorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/any23/extractor/yaml/YAMLExtractorTest.java b/core/src/test/java/org/apache/any23/extractor/yaml/YAMLExtractorTest.java
index b265c5f..f2c85ba 100644
--- a/core/src/test/java/org/apache/any23/extractor/yaml/YAMLExtractorTest.java
+++ b/core/src/test/java/org/apache/any23/extractor/yaml/YAMLExtractorTest.java
@@ -92,17 +92,4 @@ public class YAMLExtractorTest extends AbstractExtractorTestCase {
         RepositoryResult<Statement> docs = getStatements(null, null, RDF.NIL);
         Assert.assertTrue(Iterations.asList(docs).size() == 2);
     }
-
-    /**
-     * Comma separated values are parsed as well.
-     *
-     * @throws Exception
-     */
-    @Test
-    public void csvTest()
-            throws Exception {
-        assertExtract("/org/apache/any23/extractor/csv/test-comma.csv");
-        log.debug(dumpModelToTurtle());
-        assertModelNotEmpty();
-    }
 }

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/core/src/test/java/org/apache/any23/extractor/yaml/YAMLTikaParserTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/any23/extractor/yaml/YAMLTikaParserTest.java b/core/src/test/java/org/apache/any23/extractor/yaml/YAMLTikaParserTest.java
index 4727c84..680b9fa 100644
--- a/core/src/test/java/org/apache/any23/extractor/yaml/YAMLTikaParserTest.java
+++ b/core/src/test/java/org/apache/any23/extractor/yaml/YAMLTikaParserTest.java
@@ -15,11 +15,13 @@
  */
 package org.apache.any23.extractor.yaml;
 
+import com.mchange.util.AssertException;
 import java.io.InputStream;
 import org.apache.any23.mime.MIMEType;
 import org.apache.any23.mime.TikaMIMETypeDetector;
 import org.apache.any23.mime.purifier.WhiteSpacesPurifier;
 import org.junit.Assert;
+import org.junit.Before;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -32,16 +34,36 @@ public class YAMLTikaParserTest {
     private static final String file1 = "/org/apache/any23/extractor/yaml/simple-load.yml";
 
     private final Logger log = LoggerFactory.getLogger(getClass());
+    
+    private TikaMIMETypeDetector detector;
+    
+    @Before
+    public void prepareDetector() throws Exception {
+        detector = new TikaMIMETypeDetector(new WhiteSpacesPurifier());
+    }
 
-    @Test
-    public void tikaDetect()
+    /**
+     * Yaml type is detected by file name only so detector returns octet type.
+     * @throws Exception 
+     */
+    @Test(expected = AssertionError.class)
+    public void tikaStreamDetect()
             throws Exception {
         InputStream is = YAMLTikaParserTest.class.getResourceAsStream(file1);
-        TikaMIMETypeDetector detector = new TikaMIMETypeDetector(new WhiteSpacesPurifier());
         MIMEType type = detector.guessMIMEType(null, is, null);
 
         log.info("Type: {}", type.toString());
-
+        Assert.assertEquals("text/x-yaml", type.toString());
+    }
+    
+    @Test
+    public void tikaNameDetect() throws Exception {
+        String fileName = java.net.URI.create(file1).getPath();
+        
+        log.debug("normatised file name: {}", fileName);
+        MIMEType type = detector.guessMIMEType(fileName, null, null);
+        
+        log.info("Type: {}", type.toString());
         Assert.assertEquals("text/x-yaml", type.toString());
     }
 

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/csvutils/pom.xml
----------------------------------------------------------------------
diff --git a/csvutils/pom.xml b/csvutils/pom.xml
new file mode 100644
index 0000000..8f5b18d
--- /dev/null
+++ b/csvutils/pom.xml
@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <artifactId>apache-any23</artifactId>
+    <groupId>org.apache.any23</groupId>
+    <version>2.1-SNAPSHOT</version>
+    <relativePath>..</relativePath>
+  </parent>
+
+  <artifactId>apache-any23-csvutils</artifactId>
+
+  <name>Apache Any23 :: CSV Utilities</name>
+  <description>CSV specific library.</description>
+
+  <dependencies>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>apache-any23-api</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-csv</artifactId>
+    </dependency>
+    <!-- Logging -->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>${slf4j.logger.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <resources>
+      <resource>
+        <directory>${basedir}/../</directory>
+        <targetPath>META-INF</targetPath>
+        <includes>
+          <include>LICENSE.txt</include>
+          <include>NOTICE.txt</include>
+        </includes>
+      </resource>
+    </resources>
+    <pluginManagement>
+      <plugins>
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-assembly-plugin</artifactId>
+          <version>${maven-assembly-plugin.version}</version>
+          <executions>
+            <execution>
+              <id>assembly</id>
+              <phase>package</phase>
+              <goals>
+                <goal>single</goal>
+              </goals>
+            </execution>
+          </executions>
+          <configuration>
+            <attach>true</attach>
+            <skipAssembly>true</skipAssembly>
+            <tarLongFileMode>gnu</tarLongFileMode>
+          </configuration>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+  </build>
+
+  <profiles>
+    <profile>
+      <id>release</id>
+      <build>
+        <resources>
+          <resource>
+            <directory>${basedir}/../</directory>
+            <targetPath>${project.build.directory}/apidocs/META-INF</targetPath>
+            <includes>
+              <include>LICENSE.txt</include>
+              <include>NOTICE.txt</include>
+            </includes>
+          </resource>
+        </resources>
+      </build>
+    </profile>
+  </profiles>
+
+</project>

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/csvutils/src/main/java/org/apache/any23/extractor/csv/CSVReaderBuilder.java
----------------------------------------------------------------------
diff --git a/csvutils/src/main/java/org/apache/any23/extractor/csv/CSVReaderBuilder.java b/csvutils/src/main/java/org/apache/any23/extractor/csv/CSVReaderBuilder.java
new file mode 100644
index 0000000..75bb583
--- /dev/null
+++ b/csvutils/src/main/java/org/apache/any23/extractor/csv/CSVReaderBuilder.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.any23.extractor.csv;
+
+import org.apache.any23.configuration.DefaultConfiguration;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVStrategy;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * This class is responsible to build a reader first guessing the configuration
+ * from the file it self and then, if not successful, from the {@link org.apache.any23.configuration.DefaultConfiguration}.
+ *
+ * @author Davide Palmisano ( dpalmisano@gmail.com )
+ * @author Michele Mostarda ( michele.mostarda@gmail.com )
+ */
+public class CSVReaderBuilder {
+
+    private static final String DEFAULT_FIELD_DELIMITER = ",";
+
+    private static final String DEFAULT_COMMENT_DELIMITER = "#";
+
+    public static final char NULL_CHAR = ' ';
+
+    private static final char[] popularDelimiters = {'\t', '|', ',', ';'};
+
+    private static DefaultConfiguration defaultConfiguration =
+            DefaultConfiguration.singleton();
+
+    private static final CSVStrategy[] strategies;
+
+    static {
+        strategies = new CSVStrategy[ popularDelimiters.length + 1 ];
+        strategies[0] = CSVStrategy.DEFAULT_STRATEGY;
+        int index = 1;
+        for(char dlmt : popularDelimiters) {
+            strategies[index++] = getCsvStrategy(dlmt, NULL_CHAR);
+        }
+    }
+
+    /**
+     * Builds a not <code>null</code> {@link org.apache.commons.csv.CSVParser}
guessing
+     * from the provided <i>CSV</i> file.
+     *
+     * @param is {@link InputStream} of the <i>CSV</i> file where guess the configuration.
+     * @return a {@link CSVParser}
+     * @throws java.io.IOException
+     */
+    public static CSVParser build(InputStream is) throws IOException {
+        CSVStrategy bestStrategy = getBestStrategy(is);
+        if(bestStrategy == null) bestStrategy = getCSVStrategyFromConfiguration();
+        return new CSVParser( new InputStreamReader(is), bestStrategy );
+    }
+
+    /**
+     * Checks whether the given input stream is a CSV or not.
+     *
+     * @param is input stream to be verified.
+     * @return <code>true</code> if the given <code>is</code> input
stream contains a <i>CSV</i> content.
+     *         <code>false</code> otherwise.
+     * @throws IOException
+     */
+    public static boolean isCSV(InputStream is) throws IOException {
+        return getBestStrategy(is) != null;
+    }
+
+    private static CSVStrategy getBestStrategy(InputStream is) throws IOException {
+        for( CSVStrategy strategy : strategies ) {
+            if( testStrategy(is, strategy) ) {
+                return strategy;
+            }
+        }
+        return null;
+    }
+
+    private static CSVStrategy getCsvStrategy(char delimiter, char comment) {
+        return new CSVStrategy(delimiter, '\'', comment);
+    }
+
+    private static CSVStrategy getCSVStrategyFromConfiguration() {
+        char fieldDelimiter = getCharValueFromConfiguration(
+                "any23.extraction.csv.field",
+                DEFAULT_FIELD_DELIMITER
+        );
+        char commentDelimiter = getCharValueFromConfiguration(
+                "any23.extraction.csv.comment",
+                DEFAULT_COMMENT_DELIMITER
+        );
+        return new CSVStrategy(fieldDelimiter, '\'', commentDelimiter);
+    }
+
+    private static char getCharValueFromConfiguration(String property, String defaultValue)
{
+        String delimiter = defaultConfiguration.getProperty(
+                property,
+                defaultValue
+        );
+        if (delimiter.length() != 1 || delimiter.equals("")) {
+            throw new RuntimeException(property + " value must be a single character");
+        }
+        return delimiter.charAt(0);
+    }
+
+    /**
+     * make sure the reader has correct delimiter and quotation set.
+     * Check first lines and make sure they have the same amount of columns and at least
2
+     *
+     * @param is input stream to be checked
+     * @param strategy strategy to be verified.
+     * @return
+     * @throws IOException
+     * @param is
+     */
+    private static boolean testStrategy(InputStream is, CSVStrategy strategy) throws IOException
{
+        final int MIN_COLUMNS = 2;
+
+        is.mark(Integer.MAX_VALUE);
+        try {
+            final CSVParser parser = new CSVParser(new InputStreamReader(is), strategy);
+            int linesToCheck = 5;
+            int headerColumnCount = -1;
+            while (linesToCheck > 0) {
+                String[] row;
+                row = parser.getLine();
+                if (row == null) {
+                    break;
+                }
+                if (row.length < MIN_COLUMNS) {
+                    return false;
+                }
+                if (headerColumnCount == -1) { // first row
+                    headerColumnCount = row.length;
+                } else { // make sure rows have the same number of columns or one more than
the header
+                    if (row.length < headerColumnCount) {
+                        return false;
+                    } else if (row.length - 1 > headerColumnCount) {
+                        return false;
+                    }
+                }
+                linesToCheck--;
+            }
+            return true;
+        } finally {
+            is.reset();
+        }
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/csvutils/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/csvutils/src/test/resources/log4j.properties b/csvutils/src/test/resources/log4j.properties
new file mode 100644
index 0000000..a7ad0af
--- /dev/null
+++ b/csvutils/src/test/resources/log4j.properties
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootCategory=INFO, R, O  
+      
+# Stdout  
+log4j.appender.O=org.apache.log4j.ConsoleAppender  
+      
+# File  
+#log4j.appender.R=org.apache.log4j.RollingFileAppender  
+#log4j.appender.R.File=log4j.log  
+      
+# Control the maximum log file size  
+#log4j.appender.R.MaxFileSize=100KB  
+      
+# Archive log files (one backup file here)  
+log4j.appender.R.MaxBackupIndex=1  
+      
+log4j.appender.R.layout=org.apache.log4j.PatternLayout  
+log4j.appender.O.layout=org.apache.log4j.PatternLayout  
+      
+log4j.appender.R.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
 
+log4j.appender.O.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
 

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/mime/pom.xml
----------------------------------------------------------------------
diff --git a/mime/pom.xml b/mime/pom.xml
index 2014758..9db7d3b 100644
--- a/mime/pom.xml
+++ b/mime/pom.xml
@@ -38,7 +38,7 @@
     </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
-      <artifactId>apache-any23-utils</artifactId>
+      <artifactId>apache-any23-csvutils</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/mime/src/main/java/org/apache/any23/mime/TikaMIMETypeDetector.java
----------------------------------------------------------------------
diff --git a/mime/src/main/java/org/apache/any23/mime/TikaMIMETypeDetector.java b/mime/src/main/java/org/apache/any23/mime/TikaMIMETypeDetector.java
index 77955cb..0c1f80b 100644
--- a/mime/src/main/java/org/apache/any23/mime/TikaMIMETypeDetector.java
+++ b/mime/src/main/java/org/apache/any23/mime/TikaMIMETypeDetector.java
@@ -36,7 +36,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.regex.Pattern;
-import org.apache.any23.extractor.yaml.YAMLValidator;
 
 /**
  * Implementation of {@link MIMETypeDetector} based on
@@ -135,17 +134,6 @@ public class TikaMIMETypeDetector implements MIMETypeDetector {
     }
 
     /**
-     * Checks if the stream contains a valid <i>YAML</i> content.
-     *
-     * @param is
-     * @return
-     * @throws IOException
-     */
-    public static boolean checkYAMLFormat(InputStream is) throws IOException {
-        return YAMLValidator.isYAML(is);
-    }
-
-    /**
      * Tries to apply one of the given patterns on a sample of the input stream.
      *
      * @param patterns the patterns to apply.
@@ -275,8 +263,6 @@ public class TikaMIMETypeDetector implements MIMETypeDetector {
                     type = RDFFormat.TURTLE.getDefaultMIMEType();
                 } else if( checkCSVFormat(input) ) {
                     type = CSV_MIMETYPE;
-                } else if (checkYAMLFormat(input)) { // YAML detection must be at the end
-                    type = "text/x-yaml";
                 } else {
                     type = MimeTypes.OCTET_STREAM; 
                 }

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0bff3cf..dce4c53 100644
--- a/pom.xml
+++ b/pom.xml
@@ -199,7 +199,7 @@
   <modules>
     <module>api</module>
     <module>test-resources</module>
-    <module>utils</module>
+    <module>csvutils</module>
     <module>mime</module>
     <module>encoding</module>
     <module>core</module>

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/utils/pom.xml
----------------------------------------------------------------------
diff --git a/utils/pom.xml b/utils/pom.xml
deleted file mode 100644
index a6f34ec..0000000
--- a/utils/pom.xml
+++ /dev/null
@@ -1,123 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed to the Apache Software Foundation (ASF) under one or more
-  contributor license agreements.  See the NOTICE file distributed with
-  this work for additional information regarding copyright ownership.
-  The ASF licenses this file to You under the Apache License, Version 2.0
-  (the "License"); you may not use this file except in compliance with
-  the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-
-  <parent>
-    <artifactId>apache-any23</artifactId>
-    <groupId>org.apache.any23</groupId>
-    <version>2.1-SNAPSHOT</version>
-    <relativePath>..</relativePath>
-  </parent>
-
-  <artifactId>apache-any23-utils</artifactId>
-
-  <name>Apache Any23 :: Utilities</name>
-  <description>Utilities library</description>
-
-  <dependencies>
-    <dependency>
-      <groupId>${project.groupId}</groupId>
-      <artifactId>apache-any23-api</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-csv</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.yaml</groupId>
-      <artifactId>snakeyaml</artifactId>
-    </dependency>
-    <!-- Logging -->
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <version>${slf4j.logger.version}</version>
-      <scope>test</scope>
-    </dependency>
-    <!-- Testing -->
-    <dependency>
-       <groupId>junit</groupId>
-       <artifactId>junit</artifactId>
-       <scope>test</scope>
-    </dependency>
-    <dependency>
-       <groupId>${project.groupId}</groupId>
-       <artifactId>apache-any23-test-resources</artifactId>
-       <version>${project.version}</version>
-       <type>test-jar</type>
-       <scope>test</scope>
-  </dependency>
-  </dependencies>
-
-  <build>
-    <resources>
-      <resource>
-        <directory>${basedir}/../</directory>
-        <targetPath>META-INF</targetPath>
-        <includes>
-          <include>LICENSE.txt</include>
-          <include>NOTICE.txt</include>
-        </includes>
-      </resource>
-    </resources>
-    <pluginManagement>
-      <plugins>
-        <plugin>
-          <groupId>org.apache.maven.plugins</groupId>
-          <artifactId>maven-assembly-plugin</artifactId>
-          <version>${maven-assembly-plugin.version}</version>
-          <executions>
-            <execution>
-              <id>assembly</id>
-              <phase>package</phase>
-              <goals>
-                <goal>single</goal>
-              </goals>
-            </execution>
-          </executions>
-          <configuration>
-            <attach>true</attach>
-            <skipAssembly>true</skipAssembly>
-            <tarLongFileMode>gnu</tarLongFileMode>
-          </configuration>
-        </plugin>
-      </plugins>
-    </pluginManagement>
-  </build>
-
-  <profiles>
-    <profile>
-      <id>release</id>
-      <build>
-        <resources>
-          <resource>
-            <directory>${basedir}/../</directory>
-            <targetPath>${project.build.directory}/apidocs/META-INF</targetPath>
-            <includes>
-              <include>LICENSE.txt</include>
-              <include>NOTICE.txt</include>
-            </includes>
-          </resource>
-        </resources>
-      </build>
-    </profile>
-  </profiles>
-
-</project>

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/utils/src/main/java/org/apache/any23/extractor/csv/CSVReaderBuilder.java
----------------------------------------------------------------------
diff --git a/utils/src/main/java/org/apache/any23/extractor/csv/CSVReaderBuilder.java b/utils/src/main/java/org/apache/any23/extractor/csv/CSVReaderBuilder.java
deleted file mode 100644
index 75bb583..0000000
--- a/utils/src/main/java/org/apache/any23/extractor/csv/CSVReaderBuilder.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.any23.extractor.csv;
-
-import org.apache.any23.configuration.DefaultConfiguration;
-import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVStrategy;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-
-/**
- * This class is responsible to build a reader first guessing the configuration
- * from the file it self and then, if not successful, from the {@link org.apache.any23.configuration.DefaultConfiguration}.
- *
- * @author Davide Palmisano ( dpalmisano@gmail.com )
- * @author Michele Mostarda ( michele.mostarda@gmail.com )
- */
-public class CSVReaderBuilder {
-
-    private static final String DEFAULT_FIELD_DELIMITER = ",";
-
-    private static final String DEFAULT_COMMENT_DELIMITER = "#";
-
-    public static final char NULL_CHAR = ' ';
-
-    private static final char[] popularDelimiters = {'\t', '|', ',', ';'};
-
-    private static DefaultConfiguration defaultConfiguration =
-            DefaultConfiguration.singleton();
-
-    private static final CSVStrategy[] strategies;
-
-    static {
-        strategies = new CSVStrategy[ popularDelimiters.length + 1 ];
-        strategies[0] = CSVStrategy.DEFAULT_STRATEGY;
-        int index = 1;
-        for(char dlmt : popularDelimiters) {
-            strategies[index++] = getCsvStrategy(dlmt, NULL_CHAR);
-        }
-    }
-
-    /**
-     * Builds a not <code>null</code> {@link org.apache.commons.csv.CSVParser}
guessing
-     * from the provided <i>CSV</i> file.
-     *
-     * @param is {@link InputStream} of the <i>CSV</i> file where guess the configuration.
-     * @return a {@link CSVParser}
-     * @throws java.io.IOException
-     */
-    public static CSVParser build(InputStream is) throws IOException {
-        CSVStrategy bestStrategy = getBestStrategy(is);
-        if(bestStrategy == null) bestStrategy = getCSVStrategyFromConfiguration();
-        return new CSVParser( new InputStreamReader(is), bestStrategy );
-    }
-
-    /**
-     * Checks whether the given input stream is a CSV or not.
-     *
-     * @param is input stream to be verified.
-     * @return <code>true</code> if the given <code>is</code> input
stream contains a <i>CSV</i> content.
-     *         <code>false</code> otherwise.
-     * @throws IOException
-     */
-    public static boolean isCSV(InputStream is) throws IOException {
-        return getBestStrategy(is) != null;
-    }
-
-    private static CSVStrategy getBestStrategy(InputStream is) throws IOException {
-        for( CSVStrategy strategy : strategies ) {
-            if( testStrategy(is, strategy) ) {
-                return strategy;
-            }
-        }
-        return null;
-    }
-
-    private static CSVStrategy getCsvStrategy(char delimiter, char comment) {
-        return new CSVStrategy(delimiter, '\'', comment);
-    }
-
-    private static CSVStrategy getCSVStrategyFromConfiguration() {
-        char fieldDelimiter = getCharValueFromConfiguration(
-                "any23.extraction.csv.field",
-                DEFAULT_FIELD_DELIMITER
-        );
-        char commentDelimiter = getCharValueFromConfiguration(
-                "any23.extraction.csv.comment",
-                DEFAULT_COMMENT_DELIMITER
-        );
-        return new CSVStrategy(fieldDelimiter, '\'', commentDelimiter);
-    }
-
-    private static char getCharValueFromConfiguration(String property, String defaultValue)
{
-        String delimiter = defaultConfiguration.getProperty(
-                property,
-                defaultValue
-        );
-        if (delimiter.length() != 1 || delimiter.equals("")) {
-            throw new RuntimeException(property + " value must be a single character");
-        }
-        return delimiter.charAt(0);
-    }
-
-    /**
-     * make sure the reader has correct delimiter and quotation set.
-     * Check first lines and make sure they have the same amount of columns and at least
2
-     *
-     * @param is input stream to be checked
-     * @param strategy strategy to be verified.
-     * @return
-     * @throws IOException
-     * @param is
-     */
-    private static boolean testStrategy(InputStream is, CSVStrategy strategy) throws IOException
{
-        final int MIN_COLUMNS = 2;
-
-        is.mark(Integer.MAX_VALUE);
-        try {
-            final CSVParser parser = new CSVParser(new InputStreamReader(is), strategy);
-            int linesToCheck = 5;
-            int headerColumnCount = -1;
-            while (linesToCheck > 0) {
-                String[] row;
-                row = parser.getLine();
-                if (row == null) {
-                    break;
-                }
-                if (row.length < MIN_COLUMNS) {
-                    return false;
-                }
-                if (headerColumnCount == -1) { // first row
-                    headerColumnCount = row.length;
-                } else { // make sure rows have the same number of columns or one more than
the header
-                    if (row.length < headerColumnCount) {
-                        return false;
-                    } else if (row.length - 1 > headerColumnCount) {
-                        return false;
-                    }
-                }
-                linesToCheck--;
-            }
-            return true;
-        } finally {
-            is.reset();
-        }
-    }
-
-
-}

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/utils/src/main/java/org/apache/any23/extractor/yaml/YAMLValidator.java
----------------------------------------------------------------------
diff --git a/utils/src/main/java/org/apache/any23/extractor/yaml/YAMLValidator.java b/utils/src/main/java/org/apache/any23/extractor/yaml/YAMLValidator.java
deleted file mode 100644
index 5a5f63d..0000000
--- a/utils/src/main/java/org/apache/any23/extractor/yaml/YAMLValidator.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2017 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.any23.extractor.yaml;
-
-import com.google.common.collect.Iterables;
-import java.io.BufferedInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Scanner;
-import java.util.regex.Pattern;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.Yaml;
-
-/**
- * Utility class provides static methods for YAML validation.
- *
- * @author Jacek Grzebyta (grzebyta.dev [at] gmail.com)
- */
-public class YAMLValidator {
-
-    private static final Logger log = LoggerFactory.getLogger(YAMLValidator.class);
-
-    private static final Pattern YAML_PATTERN = Pattern.compile("^%YAML.*", Pattern.CASE_INSENSITIVE);
-
-    /**
-     * Detects if is contains valid YAML content.
-     * <p>
-     * In the first instance it checks if there is "%YAML" head. If not check
-     * using the brute force method by parsing input stream with yaml parser.
-     * </p>
-     * <p>
-     * NB. Only "false" results are trusted. Even if result is "true" you cannot
-     * be sure that InputStream contains YAML intentional context because
-     * comma-separated-values are pars-able by YAML parser as well.
-     * </p>
-     *
-     * @param is {@link InputStream}
-     * @return
-     * @throws IOException
-     */
-    public static boolean isYAML(InputStream is) throws IOException {
-        if (is == null) {
-            return false;
-        }
-
-        if (!is.markSupported()) {
-            is = new BufferedInputStream(is);
-        }
-
-        boolean result = false;
-
-        // mark the reading frame position. MUST BE FIRST
-        is.mark(Integer.MAX_VALUE);
-
-        while (true) {
-            // if is is empty than return false
-            if (is.available() <= 0) {
-                break;
-            }
-
-            Scanner sc = new Scanner(is);
-            String out = sc.findWithinHorizon(YAML_PATTERN, 0);
-
-            if (out != null && !out.isEmpty()) {
-                log.debug("Head: {}", out);
-                result = true;
-                break;
-            }
-            log.debug("Still not found. output is: {}", out);
-            is.reset();
-
-            try {
-                Yaml yml = new Yaml();
-                Iterable<Object> parsedOut = yml.loadAll(is);
-
-                if (Iterables.size(parsedOut) > 0) {
-                    result = true;
-                    break;
-                }
-            } catch (Exception ex) {
-                //do nothing
-            }
-
-            // final break 
-            break;
-        }
-
-        is.reset(); // MUST BE AT THE END
-        return result;
-    }
-}

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/utils/src/test/java/org/apache/any23/yaml/utils/YAMLValidatorTest.java
----------------------------------------------------------------------
diff --git a/utils/src/test/java/org/apache/any23/yaml/utils/YAMLValidatorTest.java b/utils/src/test/java/org/apache/any23/yaml/utils/YAMLValidatorTest.java
deleted file mode 100644
index fddf2fb..0000000
--- a/utils/src/test/java/org/apache/any23/yaml/utils/YAMLValidatorTest.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2017 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.any23.yaml.utils;
-
-import org.apache.any23.extractor.yaml.YAMLValidator;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.Collection;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * @author Jacek Grzebyta ( grzebyta.dev [at] gmail.com)
- */
-@RunWith(Parameterized.class)
-public class YAMLValidatorTest {
-
-    private String path;
-
-    private Boolean expected;
-
-    private Logger log = LoggerFactory.getLogger(getClass());
-
-    public YAMLValidatorTest(String path, Boolean expected) {
-        this.path = path;
-        this.expected = expected;
-    }
-
-    @Parameterized.Parameters
-    public static Collection<Object[]> getFiles() {
-        return Arrays.asList(new Object[][]{
-            {"/org/apache/any23/extractor/yaml/simple-load.yml", Boolean.TRUE},
-            {"/org/apache/any23/extractor/yaml/simple-load_no_head.yml", Boolean.TRUE},
-            {"/org/apache/any23/extractor/yaml/different-integers.yml", Boolean.TRUE},
-            {"/org/apache/any23/extractor/yaml/different-float.yml", Boolean.TRUE},
-            {"/org/apache/any23/extractor/csv/test-comma.csv", Boolean.TRUE}});
-    }
-
-    @Test
-    public void runTest()
-            throws Exception {
-        log.info("Try path: {}", path);
-        InputStream is = YAMLValidatorTest.class.getResourceAsStream(path);
-        boolean result = YAMLValidator.isYAML(is);
-        log.debug("Test resutl: {}", result);
-        Assert.assertSame(expected, result);
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/any23/blob/9839e212/utils/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/utils/src/test/resources/log4j.properties b/utils/src/test/resources/log4j.properties
deleted file mode 100644
index 3860396..0000000
--- a/utils/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# log4j.rootCategory=INFO, R, O  
-log4j.rootCategory=INFO, O  
-      
-# Stdout  
-log4j.appender.O=org.apache.log4j.ConsoleAppender  
-      
-# File  
-#log4j.appender.R=org.apache.log4j.RollingFileAppender  
-#log4j.appender.R.File=log4j.log  
-      
-# Control the maximum log file size  
-#log4j.appender.R.MaxFileSize=100KB  
-      
-# Archive log files (one backup file here)  
-log4j.appender.R.MaxBackupIndex=1  
-      
-log4j.appender.R.layout=org.apache.log4j.PatternLayout  
-log4j.appender.O.layout=org.apache.log4j.PatternLayout  
-      
-log4j.appender.R.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
 
-log4j.appender.O.layout.ConversionPattern=[%d{ISO8601}]%5p%6.6r[%t]%x - %C.%M(%F:%L) - %m%n
 


Mime
View raw message