drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] vvysotskyi commented on a change in pull request #1667: DRILL-7069: Moved version checks outside loops in transformBinaryInMetadataCache
Date Fri, 01 Mar 2019 11:36:10 GMT
vvysotskyi commented on a change in pull request #1667: DRILL-7069: Moved version checks outside
loops in transformBinaryInMetadataCache
URL: https://github.com/apache/drill/pull/1667#discussion_r261519267
 
 

 ##########
 File path: exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetReaderUtility.java
 ##########
 @@ -292,30 +294,47 @@ public static void transformBinaryInMetadataCache(ParquetTableMetadataBase
parqu
     Set<List<String>> columnsNames = getBinaryColumnsNames(parquetTableMetadata);
     boolean allowBinaryMetadata = allowBinaryMetadata(parquetTableMetadata.getDrillVersion(),
readerConfig);
 
+    // Setting Min / Max values for ParquetTableMetadata_v1
+    if (new MetadataVersion(1, 0).equals(new MetadataVersion(parquetTableMetadata.getMetadataVersion())))
{
+      for (ParquetFileMetadata file : parquetTableMetadata.getFiles()) {
+        for (RowGroupMetadata rowGroupMetadata : file.getRowGroups()) {
+          Long rowCount = rowGroupMetadata.getRowCount();
+          for (ColumnMetadata columnMetadata : rowGroupMetadata.getColumns()) {
+              if (columnMetadata.getPrimitiveType() == PrimitiveTypeName.BINARY || columnMetadata.getPrimitiveType()
== PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
+                setMinMaxValues(columnMetadata, rowCount, allowBinaryMetadata, false);
+              }
+          }
+        }
+      }
+      return;
+    }
+
+    // Variables needed for debugging only
+    Stopwatch timer = logger.isDebugEnabled() ? Stopwatch.createStarted() : null;
+    int maxRowGroups = 0, minRowGroups = 99999, maxNumColumns = 0;
+
+    // Setting Min / Max values for V2 and V3 versions; for versions V3_3 and above need
to do decoding
+    boolean needDecoding = new MetadataVersion(parquetTableMetadata.getMetadataVersion()).compareTo(new
MetadataVersion(3, 3)) >= 0;
     for (ParquetFileMetadata file : parquetTableMetadata.getFiles()) {
+      maxRowGroups = Math.max(maxRowGroups,file.getRowGroups().size());
+      minRowGroups = Math.min(minRowGroups,file.getRowGroups().size());
       for (RowGroupMetadata rowGroupMetadata : file.getRowGroups()) {
         Long rowCount = rowGroupMetadata.getRowCount();
+        maxNumColumns = Math.max(maxNumColumns,rowGroupMetadata.getColumns().size());
         for (ColumnMetadata columnMetadata : rowGroupMetadata.getColumns()) {
-          // Setting Min / Max values for ParquetTableMetadata_v1
-          if (new MetadataVersion(1, 0).equals(new MetadataVersion(parquetTableMetadata.getMetadataVersion())))
{
-            if (columnMetadata.getPrimitiveType() == PrimitiveTypeName.BINARY
-                || columnMetadata.getPrimitiveType() == PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
{
-              setMinMaxValues(columnMetadata, rowCount, allowBinaryMetadata, false);
-            }
-          }
-          // Setting Min / Max values for V2 and all V3 versions prior to V3_3
-          else if (new MetadataVersion(parquetTableMetadata.getMetadataVersion()).compareTo(new
MetadataVersion(3, 3)) < 0
-                    && columnsNames.contains(Arrays.asList(columnMetadata.getName())))
{
-            setMinMaxValues(columnMetadata, rowCount, allowBinaryMetadata, false);
-          }
-          // Setting Min / Max values for V3_3 and all next versions
-          else if (new MetadataVersion(parquetTableMetadata.getMetadataVersion()).compareTo(new
MetadataVersion(3, 3)) >= 0
-                      && columnsNames.contains(Arrays.asList(columnMetadata.getName())))
{
-            setMinMaxValues(columnMetadata, rowCount, allowBinaryMetadata, true);
+           if (columnsNames.contains(Arrays.asList(columnMetadata.getName()))) {
+            setMinMaxValues(columnMetadata, rowCount, allowBinaryMetadata, needDecoding);
           }
         }
       }
     }
+
+    if (timer != null) { // log a debug message and stop the timer
+      String reportRG = 1 == maxRowGroups ? "1 rowgroup" : "between " + minRowGroups + "-"
+ maxRowGroups + "rowgroups";
+      logger.debug("Transforming binary in metadata cache took {} ms ({} files, {} per file,
max {} columns)", timer.elapsed(TimeUnit.MILLISECONDS),
+        parquetTableMetadata.getFiles().size(),reportRG,maxNumColumns);
 
 Review comment:
   And please add spaces here: `parquetTableMetadata.getFiles().size(),reportRG,maxNumColumns);`
-> `parquetTableMetadata.getFiles().size(), reportRG, maxNumColumns);`

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message