carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [08/47] incubator-carbondata git commit: [CARBONDATA-84] Change lock framework to accept locklocation (#844)
Date Mon, 01 Aug 2016 10:05:06 GMT
[CARBONDATA-84] Change lock framework to accept locklocation (#844)

1.get dictionary lock location from CarbonTablePath
2.Change lock framework to accept lock location

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/584402e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/584402e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/584402e8

Branch: refs/heads/master
Commit: 584402e8008d64b57233dc5be2a13a2739e32c61
Parents: 07116d2
Author: ashokblend <ashok.blend@gmail.com>
Authored: Sat Jul 23 11:06:21 2016 +0530
Committer: Venkata Ramana G <g.ramana.v1@gmail.com>
Committed: Sat Jul 23 11:06:21 2016 +0530

----------------------------------------------------------------------
 .../core/carbon/path/CarbonStorePath.java       |  2 +-
 .../core/carbon/path/CarbonTablePath.java       | 18 +++++++++++++++-
 .../core/locks/CarbonLockFactory.java           | 22 ++++++++++++++++++++
 .../org/carbondata/core/locks/HdfsFileLock.java | 16 ++++++++++----
 .../carbondata/core/locks/LocalFileLock.java    | 16 ++++++++++----
 .../carbondata/core/locks/ZooKeeperLocking.java | 18 ++++++++--------
 .../spark/rdd/CarbonGlobalDictionaryRDD.scala   |  9 ++++++--
 7 files changed, 80 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java b/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
index c393f43..eac98e6 100644
--- a/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
+++ b/core/src/main/java/org/carbondata/core/carbon/path/CarbonStorePath.java
@@ -41,7 +41,7 @@ public class CarbonStorePath extends Path {
    */
   public static CarbonTablePath getCarbonTablePath(String storePath,
       CarbonTableIdentifier tableIdentifier) {
-    CarbonTablePath carbonTablePath = new CarbonTablePath(
+    CarbonTablePath carbonTablePath = new CarbonTablePath(tableIdentifier,
         storePath + File.separator + tableIdentifier.getDatabaseName() + File.separator
             + tableIdentifier.getTableName());
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
index 9ac8802..bd2c472 100644
--- a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
@@ -20,6 +20,7 @@ package org.carbondata.core.carbon.path;
 
 import java.io.File;
 
+import org.carbondata.core.carbon.CarbonTableIdentifier;
 import org.carbondata.core.constants.CarbonCommonConstants;
 import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
 import org.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
@@ -50,9 +51,16 @@ public class CarbonTablePath extends Path {
   protected static final String INDEX_FILE_EXT = ".carbonindex";
 
   protected String tablePath;
+  protected CarbonTableIdentifier carbonTableIdentifier;
 
-  public CarbonTablePath(String tablePathString) {
+  /**
+   *
+   * @param carbonTableIdentifier
+   * @param tablePathString
+   */
+  public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String tablePathString)
{
     super(tablePathString);
+    this.carbonTableIdentifier = carbonTableIdentifier;
     this.tablePath = tablePathString;
   }
 
@@ -127,6 +135,14 @@ public class CarbonTablePath extends Path {
   }
 
   /**
+   * @return it return relative directory
+   */
+  public String getRelativeDictionaryDirectory() {
+    return carbonTableIdentifier.getDatabaseName() + File.separator + carbonTableIdentifier
+        .getTableName();
+  }
+
+  /**
    * This method will return the metadata directory location for a table
    *
    * @return

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
index dcf2c2c..8616c68 100644
--- a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
+++ b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java
@@ -62,6 +62,28 @@ public class CarbonLockFactory {
   }
 
   /**
+   *
+   * @param locFileLocation
+   * @param lockFile
+   * @return carbon lock
+   */
+  public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) {
+    switch (lockTypeConfigured.toUpperCase()) {
+      case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
+        return new LocalFileLock(locFileLocation, lockFile);
+
+      case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
+        return new ZooKeeperLocking(locFileLocation, lockFile);
+
+      case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
+        return new HdfsFileLock(locFileLocation, lockFile);
+
+      default:
+        throw new UnsupportedOperationException("Not supported the lock type");
+    }
+  }
+
+  /**
    * This method will set the zookeeper status whether zookeeper to be used for locking or
not.
    */
   private static void updateZooKeeperLockingStatus() {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
index e1fdd73..e8f44ed 100644
--- a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java
@@ -45,14 +45,22 @@ public class HdfsFileLock extends AbstractCarbonLock {
   }
 
   /**
+   * @param lockFileLocation
+   * @param lockFile
+   */
+  public HdfsFileLock(String lockFileLocation, String lockFile) {
+    this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation
+        + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+    initRetry();
+  }
+
+  /**
    * @param tableIdentifier
    * @param lockFile
    */
   public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
-    this.location =
-        tmpPath + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
-            + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getTableName()
-            + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+    this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+        .getTableName(), lockFile);
     initRetry();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
index 8194bfd..1120091 100644
--- a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java
@@ -73,14 +73,22 @@ public class LocalFileLock extends AbstractCarbonLock {
   }
 
   /**
+   * @param lockFileLocation
+   * @param lockFile
+   */
+  public LocalFileLock(String lockFileLocation, String lockFile) {
+    this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation;
+    this.lockFile = lockFile;
+    initRetry();
+  }
+
+  /**
    * @param tableIdentifier
    * @param lockFile
    */
   public LocalFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
-    this.location =
-        tmpPath + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
-            + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getTableName();
-    this.lockFile = lockFile;
+    this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+        .getTableName(), lockFile);
     initRetry();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
index 47349ad..e1db3c6 100644
--- a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
+++ b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java
@@ -68,22 +68,22 @@ public class ZooKeeperLocking extends AbstractCarbonLock {
 
   private String lockTypeFolder;
 
+  public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
+    this(tableIdentifier.getDatabaseName() + '.' + tableIdentifier.getTableName(), lockFile);
+  }
+
   /**
-   * @param tableIdentifier
+   * @param lockLocation
    * @param lockFile
    */
-  public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
+  public ZooKeeperLocking(String lockLocation, String lockFile) {
     this.lockName = lockFile;
-    this.tableIdFolder =
-        zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
-            + '.' + tableIdentifier.getTableName();
+    this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
 
     zk = ZookeeperInit.getInstance().getZookeeper();
 
-    this.lockTypeFolder =
-        zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier.getDatabaseName()
-            + '.' + tableIdentifier.getTableName() + CarbonCommonConstants.FILE_SEPARATOR
-            + lockFile;
+    this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
+        + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
     try {
       createBaseNode();
       // if exists returns null then path doesnt exist. so creating.

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/584402e8/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 9abcceb..e519030 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -30,6 +30,7 @@ import org.apache.spark._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.Row
 
+import org.carbondata.common.factory.CarbonCommonFactory
 import org.carbondata.common.logging.LogServiceFactory
 import org.carbondata.core.carbon.{CarbonTableIdentifier, ColumnIdentifier}
 import org.carbondata.core.carbon.metadata.datatype.DataType
@@ -295,8 +296,12 @@ class CarbonGlobalDictionaryGenerateRDD(
       var dictionaryForDistinctValueLookUp: org.carbondata.core.cache.dictionary.Dictionary
= _
       var dictionaryForSortIndexWriting: org.carbondata.core.cache.dictionary.Dictionary
= _
       var dictionaryForDistinctValueLookUpCleared: Boolean = false
-      val dictLock = CarbonLockFactory.getCarbonLockObj(model.table,
-        model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
+      val pathService = CarbonCommonFactory.getPathService
+      val carbonTablePath = pathService.getCarbonTablePath(model.columnIdentifier(split.index),
+          model.hdfsLocation, model.table)
+      val dictLock = CarbonLockFactory
+        .getCarbonLockObj(carbonTablePath.getRelativeDictionaryDirectory,
+          model.columnIdentifier(split.index).getColumnId + LockUsage.LOCK)
       // generate distinct value list
       try {
         val t1 = System.currentTimeMillis


Mime
View raw message