carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-2108]Updated unsafe sort memory configuration
Date Fri, 02 Feb 2018 17:48:35 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 2610a6091 -> 27ec6515a


[CARBONDATA-2108]Updated unsafe sort memory configuration

Deprecated old property: sort.inmemory.size.inmb
Added new property: carbon.sort.storage.inmemory.size.inmb,
If user has configured old property then internally it will be converted to new property
for ex: If user has configured sort.inmemory.size.inmb then 20% memory will be used as working
memory and rest for storage memory

This closes #1896


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/27ec6515
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/27ec6515
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/27ec6515

Branch: refs/heads/master
Commit: 27ec6515a143dc3b697ac914bfcd4cfe10a49e17
Parents: 2610a60
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Wed Jan 31 18:43:02 2018 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Fri Feb 2 23:18:21 2018 +0530

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |  5 +
 .../core/memory/UnsafeMemoryManager.java        |  2 +-
 .../core/memory/UnsafeSortMemoryManager.java    |  6 +-
 .../carbondata/core/util/CarbonProperties.java  | 99 ++++++++++++++++++++
 4 files changed, 108 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 87eec8a..8480758 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1585,6 +1585,11 @@ public final class CarbonCommonConstants {
 
   public static final String CARBON_ENABLE_PAGE_LEVEL_READER_IN_COMPACTION_DEFAULT = "true";
 
+  @CarbonProperty
+  public static final String IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB =
+      "carbon.sort.storage.inmemory.size.inmb";
+  public static final String IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT = "512";
+
   private CarbonCommonConstants() {
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
index 4222e14..d3b9b48 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
@@ -47,7 +47,7 @@ public class UnsafeMemoryManager {
           .getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
               CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT));
     } catch (Exception e) {
-      size = Long.parseLong(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+      size = Long.parseLong(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT);
       LOGGER.info("Wrong memory size given, "
           + "so setting default value to " + size);
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
index c63b320..67bb6cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeSortMemoryManager.java
@@ -75,10 +75,10 @@ public class UnsafeSortMemoryManager {
     long size;
     try {
       size = Long.parseLong(CarbonProperties.getInstance()
-          .getProperty(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB,
-              CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT));
+          .getProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB,
+              CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT));
     } catch (Exception e) {
-      size = Long.parseLong(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+      size = Long.parseLong(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT);
       LOGGER.info("Wrong memory size given, " + "so setting default value to " + size);
     }
     if (size < 1024) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/27ec6515/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 39a0b80..3dc7b8f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -223,6 +223,9 @@ public final class CarbonProperties {
     validateSortIntermediateFilesLimit();
     validateEnableAutoHandoff();
     validateSchedulerMinRegisteredRatio();
+    validateSortMemorySizeInMB();
+    validateWorkingMemory();
+    validateSortStorageMemory();
   }
 
   /**
@@ -1252,4 +1255,100 @@ public final class CarbonProperties {
   public void addPropertyToPropertySet(Set<String> externalPropertySet) {
     propertySet.addAll(externalPropertySet);
   }
+
+  private void validateSortMemorySizeInMB() {
+    int sortMemorySizeInMBDefault =
+        Integer.parseInt(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+    int sortMemorySizeInMB = 0;
+    try {
+      sortMemorySizeInMB = Integer.parseInt(
+          carbonProperties.getProperty(CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB));
+    } catch (NumberFormatException e) {
+      LOGGER.error(
+          "The specified value for property " + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB
+              + "is Invalid." + " Taking the default value."
+              + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+      sortMemorySizeInMB = sortMemorySizeInMBDefault;
+    }
+    if (sortMemorySizeInMB < sortMemorySizeInMBDefault) {
+      LOGGER.error(
+          "The specified value for property " + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB
+              + "is less than default value." + ". Taking the default value."
+              + CarbonCommonConstants.IN_MEMORY_FOR_SORT_DATA_IN_MB_DEFAULT);
+      sortMemorySizeInMB = sortMemorySizeInMBDefault;
+    }
+    String unsafeWorkingMemoryString =
+        carbonProperties.getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB);
+    String unsafeSortStorageMemoryString =
+        carbonProperties.getProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB);
+    int workingMemory = 512;
+    int sortStorageMemory;
+    if (null == unsafeWorkingMemoryString && null == unsafeSortStorageMemoryString)
{
+      workingMemory = workingMemory > ((sortMemorySizeInMB * 20) / 100) ?
+          workingMemory :
+          ((sortMemorySizeInMB * 20) / 100);
+      sortStorageMemory = sortMemorySizeInMB - workingMemory;
+      carbonProperties
+          .setProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, workingMemory +
"");
+      carbonProperties.setProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB,
+          sortStorageMemory + "");
+    } else if (null != unsafeWorkingMemoryString && null == unsafeSortStorageMemoryString)
{
+      carbonProperties.setProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB,
+          sortMemorySizeInMB + "");
+    } else if (null == unsafeWorkingMemoryString && null != unsafeSortStorageMemoryString)
{
+      carbonProperties
+          .setProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, sortMemorySizeInMB
+ "");
+    }
+  }
+
+  private void validateWorkingMemory() {
+    int unsafeWorkingMemoryDefault =
+        Integer.parseInt(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT);
+    int unsafeWorkingMemory = 0;
+    try {
+      unsafeWorkingMemory = Integer.parseInt(
+          carbonProperties.getProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB));
+    } catch (NumberFormatException e) {
+      LOGGER.error("The specified value for property "
+          + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT + "is invalid."
+          + " Taking the default value."
+          + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT);
+      unsafeWorkingMemory = unsafeWorkingMemoryDefault;
+    }
+    if (unsafeWorkingMemory < unsafeWorkingMemoryDefault) {
+      LOGGER.error("The specified value for property "
+          + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT
+          + "is less than the default value." + ". Taking the default value."
+          + CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT);
+      unsafeWorkingMemory = unsafeWorkingMemoryDefault;
+    }
+    carbonProperties
+        .setProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, unsafeWorkingMemory
+ "");
+  }
+
+  private void validateSortStorageMemory() {
+    int unsafeSortStorageMemoryDefault =
+        Integer.parseInt(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT);
+    int unsafeSortStorageMemory = 0;
+    try {
+      unsafeSortStorageMemory = Integer.parseInt(carbonProperties
+          .getProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB));
+    } catch (NumberFormatException e) {
+      LOGGER.error("The specified value for property "
+          + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB + "is invalid."
+          + " Taking the default value."
+          + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT);
+      unsafeSortStorageMemory = unsafeSortStorageMemoryDefault;
+    }
+    if (unsafeSortStorageMemory < unsafeSortStorageMemoryDefault) {
+      LOGGER.error("The specified value for property "
+          + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB
+          + "is less than the default value." + " Taking the default value."
+          + CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB_DEFAULT);
+      unsafeSortStorageMemory = unsafeSortStorageMemoryDefault;
+    }
+    carbonProperties.setProperty(CarbonCommonConstants.IN_MEMORY_STORAGE_FOR_SORTED_DATA_IN_MB,
+        unsafeSortStorageMemory + "");
+  }
+
 }


Mime
View raw message