carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [48/52] [partial] incubator-carbondata git commit: Renamed packages to org.apache.carbondata and fixed errors
Date Mon, 15 Aug 2016 07:09:33 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
new file mode 100644
index 0000000..03a6d6a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnReverseDictionaryInfo.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import net.jpountz.xxhash.XXHash32;
+import net.jpountz.xxhash.XXHashFactory;
+
+/**
+ * class that implements methods specific for dictionary data look up
+ */
+public class ColumnReverseDictionaryInfo extends AbstractColumnDictionaryInfo {
+
+  /**
+   * Map which will maintain mapping of byte array to surrogate key
+   */
+  private Map<DictionaryByteArrayWrapper, Integer> dictionaryByteArrayToSurrogateKeyMap;
+
+  /**
+   * hashing algorithm to calculate hash code
+   */
+  private XXHash32 xxHash32;
+
+  /**
+   * check and initialize xxHash32 if enabled
+   */
+  public ColumnReverseDictionaryInfo() {
+    boolean useXXHash = Boolean.valueOf(CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.ENABLE_XXHASH,
+            CarbonCommonConstants.ENABLE_XXHASH_DEFAULT));
+    if (useXXHash) {
+      xxHash32 = XXHashFactory.fastestInstance().hash32();
+    }
+  }
+
+  /**
+   * This method will find and return the surrogate key for a given dictionary value
+   * Applicable scenario:
+   * 1. Incremental data load : Dictionary will not be generated for existing values. For
+   * that values have to be looked up in the existing dictionary cache.
+   * 2. Filter scenarios where from value surrogate key has to be found.
+   *
+   * @param value dictionary value as byte array. It will be treated as key here
+   * @return if found returns key else 0
+   */
+  @Override public int getSurrogateKey(byte[] value) {
+    DictionaryByteArrayWrapper dictionaryByteArrayWrapper =
+        new DictionaryByteArrayWrapper(value, xxHash32);
+    Integer surrogateKeyInMap =
+        dictionaryByteArrayToSurrogateKeyMap.get(dictionaryByteArrayWrapper);
+    if (null == surrogateKeyInMap) {
+      return CarbonCommonConstants.INVALID_SURROGATE_KEY;
+    }
+    return surrogateKeyInMap;
+  }
+
+  /**
+   * This method will add a new dictionary chunk to existing list of dictionary chunks
+   *
+   * @param dictionaryChunk
+   */
+  @Override public void addDictionaryChunk(List<byte[]> dictionaryChunk) {
+    dictionaryChunks.add(dictionaryChunk);
+    if (null == dictionaryByteArrayToSurrogateKeyMap) {
+      createDictionaryByteArrayToSurrogateKeyMap(dictionaryChunk.size());
+    }
+    addDataToDictionaryMap();
+  }
+
+  /**
+   * This method will add the new dictionary data to map
+   */
+  private void addDataToDictionaryMap() {
+    int surrogateKey = dictionaryByteArrayToSurrogateKeyMap.size();
+    List<byte[]> oneDictionaryChunk = dictionaryChunks.get(dictionaryChunks.size() - 1);
+    for (int i = 0; i < oneDictionaryChunk.size(); i++) {
+      // create a wrapper class that will calculate hash code for byte array
+      DictionaryByteArrayWrapper dictionaryByteArrayWrapper =
+          new DictionaryByteArrayWrapper(oneDictionaryChunk.get(i), xxHash32);
+      dictionaryByteArrayToSurrogateKeyMap.put(dictionaryByteArrayWrapper, ++surrogateKey);
+    }
+  }
+
+  /**
+   * This method will create the dictionary map. First time it will
+   * create dictionary map with capacity equal to list of byte arrays
+   *
+   * @param initialMapSize capacity to which map is to be instantiated
+   */
+  private void createDictionaryByteArrayToSurrogateKeyMap(int initialMapSize) {
+    dictionaryByteArrayToSurrogateKeyMap = new ConcurrentHashMap<>(initialMapSize);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
new file mode 100644
index 0000000..d29be86
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/Dictionary.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+/**
+ * dictionary interface which declares methods for finding surrogate key for a
+ * given dictionary value and finding dictionary value from a given surrogate key
+ */
+public interface Dictionary {
+
+  /**
+   * This method will find and return the surrogate key for a given dictionary value
+   * Applicable scenario:
+   * 1. Incremental data load : Dictionary will not be generated for existing values. For
+   * that values have to be looked up in the existing dictionary cache.
+   * 2. Filter scenarios where from value surrogate key has to be found.
+   *
+   * @param value dictionary value
+   * @return if found returns key else 0
+   */
+  int getSurrogateKey(String value);
+
+  /**
+   * This method will find and return the surrogate key for a given dictionary value
+   * Applicable scenario:
+   * 1. Incremental data load : Dictionary will not be generated for existing values. For
+   * that values have to be looked up in the existing dictionary cache.
+   * 2. Filter scenarios where from value surrogate key has to be found.
+   *
+   * @param value dictionary value as byte array
+   * @return if found returns key else -1
+   */
+  int getSurrogateKey(byte[] value);
+
+  /**
+   * This method will find and return the dictionary value for a given surrogate key.
+   * Applicable scenarios:
+   * 1. Query final result preparation : While convert the final result which will
+   * be surrogate key back to original dictionary values this method will be used
+   *
+   * @param surrogateKey a unique ID for a dictionary value
+   * @return value if found else null
+   */
+  String getDictionaryValueForKey(int surrogateKey);
+
+  /**
+   * This method will find and return the sort index for a given dictionary id.
+   * Applicable scenarios:
+   * 1. Used in case of order by queries when data sorting is required
+   *
+   * @param surrogateKey a unique ID for a dictionary value
+   * @return if found returns key else 0
+   */
+  int getSortedIndex(int surrogateKey);
+
+  /**
+   * This method will find and return the dictionary value from sorted index.
+   * Applicable scenarios:
+   * 1. Query final result preparation in case of order by queries:
+   * While convert the final result which will
+   * be surrogate key back to original dictionary values this method will be used
+   *
+   * @param sortedIndex sort index of dictionary value
+   * @return value if found else null
+   */
+  String getDictionaryValueFromSortedIndex(int sortedIndex);
+
+  /**
+   * The method return the dictionary chunks wrapper of a column
+   * The wrapper wraps the list<list<bye[]>> and provide the iterator to retrieve the chunks
+   * members.
+   * Applications Scenario:
+   * For preparing the column Sort info while writing the sort index file.
+   *
+   * @return
+   */
+  DictionaryChunksWrapper getDictionaryChunks();
+
+  /**
+   * This method will release the objects and set default value for primitive types
+   */
+  void clear();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryByteArrayWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryByteArrayWrapper.java
new file mode 100644
index 0000000..73f7702
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryByteArrayWrapper.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+import net.jpountz.xxhash.XXHash32;
+
+/**
+ * class that holds the byte array and overrides equals and hashcode method which
+ * will be useful for object comparison
+ */
+public class DictionaryByteArrayWrapper {
+
+  /**
+   * dictionary value as byte array
+   */
+  private byte[] data;
+
+  /**
+   * hashing algorithm to calculate hash code
+   */
+  private XXHash32 xxHash32;
+
+  /**
+   * @param data
+   */
+  public DictionaryByteArrayWrapper(byte[] data) {
+    this.data = data;
+  }
+
+  /**
+   * @param data
+   * @param xxHash32
+   */
+  public DictionaryByteArrayWrapper(byte[] data, XXHash32 xxHash32) {
+    this(data);
+    this.xxHash32 = xxHash32;
+  }
+
+  /**
+   * This method will compare 2 DictionaryByteArrayWrapper objects
+   *
+   * @param other
+   * @return
+   */
+  @Override public boolean equals(Object other) {
+    if (this == other) {
+      return true;
+    }
+    if (other == null || getClass() != other.getClass()) {
+      return false;
+    }
+    DictionaryByteArrayWrapper otherObjectToCompare = (DictionaryByteArrayWrapper) other;
+    if (data.length != otherObjectToCompare.data.length) {
+      return false;
+    }
+    return ByteUtil.UnsafeComparer.INSTANCE.equals(data, otherObjectToCompare.data);
+
+  }
+
+  /**
+   * This method will calculate the hash code for given data
+   *
+   * @return
+   */
+  @Override public int hashCode() {
+    if (null != xxHash32) {
+      return xxHash32.hash(data, 0, data.length, 0);
+    }
+    int result = Arrays.hashCode(data);
+    result = 31 * result;
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
new file mode 100644
index 0000000..8da437b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.carbon.ColumnIdentifier;
+
+public interface DictionaryCacheLoader {
+
+  /**
+   * This method will load the dictionary data for a given columnIdentifier
+   *
+   * @param dictionaryInfo             dictionary info object which will hold the required data
+   *                                   for a given column
+   * @param columnIdentifier           column unique identifier
+   * @param dictionaryChunkStartOffset start offset from where dictionary file has to
+   *                                   be read
+   * @param dictionaryChunkEndOffset   end offset till where dictionary file has to
+   *                                   be read
+   * @param loadSortIndex              flag to indicate whether the sort index file has to be
+   *                                   read in memory after dictionary loading
+   * @throws IOException
+   */
+  void load(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier,
+      long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
+      throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
new file mode 100644
index 0000000..6e603f9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
+import org.apache.carbondata.core.carbon.ColumnIdentifier;
+import org.apache.carbondata.core.reader.CarbonDictionaryReader;
+import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
+import org.apache.carbondata.core.service.DictionaryService;
+
+/**
+ * This class is responsible for loading the dictionary data for given columns
+ */
+public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
+
+  /**
+   * carbon table identifier
+   */
+  private CarbonTableIdentifier carbonTableIdentifier;
+
+  /**
+   * carbon store path
+   */
+  private String carbonStorePath;
+
+  /**
+   * @param carbonTableIdentifier fully qualified table name
+   * @param carbonStorePath       hdfs store path
+   */
+  public DictionaryCacheLoaderImpl(CarbonTableIdentifier carbonTableIdentifier,
+      String carbonStorePath) {
+    this.carbonTableIdentifier = carbonTableIdentifier;
+    this.carbonStorePath = carbonStorePath;
+  }
+
+  /**
+   * This method will load the dictionary data for a given columnIdentifier
+   *
+   * @param dictionaryInfo             dictionary info object which will hold the required data
+   *                                   for a given column
+   * @param columnIdentifier           column unique identifier
+   * @param dictionaryChunkStartOffset start offset from where dictionary file has to
+   *                                   be read
+   * @param dictionaryChunkEndOffset   end offset till where dictionary file has to
+   *                                   be read
+   * @param loadSortIndex              flag to indicate whether the sort index file has to be
+   *                                   read in memory after dictionary loading
+   * @throws IOException
+   */
+  @Override public void load(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier,
+      long dictionaryChunkStartOffset, long dictionaryChunkEndOffset, boolean loadSortIndex)
+      throws IOException {
+    List<byte[]> dictionaryChunk =
+        load(columnIdentifier, dictionaryChunkStartOffset, dictionaryChunkEndOffset);
+    if (loadSortIndex) {
+      readSortIndexFile(dictionaryInfo, columnIdentifier);
+    }
+    dictionaryInfo.addDictionaryChunk(dictionaryChunk);
+  }
+
+  /**
+   * This method will load the dictionary data between a given start and end offset
+   *
+   * @param columnIdentifier column unique identifier
+   * @param startOffset      start offset of dictionary file
+   * @param endOffset        end offset of dictionary file
+   * @return list of dictionary value
+   * @throws IOException
+   */
+  private List<byte[]> load(ColumnIdentifier columnIdentifier, long startOffset, long endOffset)
+      throws IOException {
+    CarbonDictionaryReader dictionaryReader = getDictionaryReader(columnIdentifier);
+    List<byte[]> dictionaryValue = null;
+    try {
+      dictionaryValue = dictionaryReader.read(startOffset, endOffset);
+    } finally {
+      dictionaryReader.close();
+    }
+    return dictionaryValue;
+  }
+
+  /**
+   * This method will read the sort index file and load into memory
+   *
+   * @param dictionaryInfo
+   * @param columnIdentifier
+   * @throws IOException
+   */
+  private void readSortIndexFile(DictionaryInfo dictionaryInfo, ColumnIdentifier columnIdentifier)
+      throws IOException {
+    CarbonDictionarySortIndexReader sortIndexReader = getSortIndexReader(columnIdentifier);
+    try {
+      dictionaryInfo.setSortOrderIndex(sortIndexReader.readSortIndex());
+      dictionaryInfo.setSortReverseOrderIndex(sortIndexReader.readInvertedSortIndex());
+    } finally {
+      sortIndexReader.close();
+    }
+  }
+
+  /**
+   * This method will create a dictionary reader instance to read the dictionary file
+   *
+   * @param columnIdentifier unique column identifier
+   * @return carbon dictionary reader instance
+   */
+  private CarbonDictionaryReader getDictionaryReader(ColumnIdentifier columnIdentifier) {
+    DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
+    return dictService
+        .getDictionaryReader(carbonTableIdentifier, columnIdentifier, carbonStorePath);
+  }
+
+  /**
+   * @param columnIdentifier unique column identifier
+   * @return sort index reader instance
+   */
+  private CarbonDictionarySortIndexReader getSortIndexReader(ColumnIdentifier columnIdentifier) {
+    DictionaryService dictService = CarbonCommonFactory.getDictionaryService();
+    return dictService
+        .getDictionarySortIndexReader(carbonTableIdentifier, columnIdentifier, carbonStorePath);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryChunksWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryChunksWrapper.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryChunksWrapper.java
new file mode 100644
index 0000000..c0c2096
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryChunksWrapper.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * The wrapper class wraps the list<list<bye[]>> and provide the iterator to retrieve the chunks
+ * members and expose the getSize API to get size of members in the List<List<byte>> chunks.
+ * Applications Scenario:
+ * For preparing the column Sort info while writing the sort index file.
+ */
+public class DictionaryChunksWrapper implements Iterator<byte[]> {
+
+  /**
+   * list of dictionaryChunks
+   */
+  private List<List<byte[]>> dictionaryChunks;
+
+  /**
+   * size of the list
+   */
+  private int size;
+
+  /**
+   * Current index of the list
+   */
+  private int currentIndex;
+
+  /**
+   * variable holds the count of elements already iterated
+   */
+  private int iteratorIndex;
+
+  /**
+   * variable holds the current index of List<List<byte[]>> being traversed
+   */
+  private int outerIndex;
+
+  /**
+   * Constructor of DictionaryChunksWrapper
+   *
+   * @param dictionaryChunks
+   */
+  public DictionaryChunksWrapper(List<List<byte[]>> dictionaryChunks) {
+    this.dictionaryChunks = dictionaryChunks;
+    for (List<byte[]> chunk : dictionaryChunks) {
+      this.size += chunk.size();
+    }
+  }
+
+  /**
+   * Returns {@code true} if the iteration has more elements.
+   * (In other words, returns {@code true} if {@link #next} would
+   * return an element rather than throwing an exception.)
+   *
+   * @return {@code true} if the iteration has more elements
+   */
+  @Override public boolean hasNext() {
+    return (currentIndex < size);
+  }
+
+  /**
+   * Returns the next element in the iteration.
+   * The method pics the next elements from the first inner list till first is not finished, pics
+   * the second inner list ...
+   *
+   * @return the next element in the iteration
+   */
+  @Override public byte[] next() {
+    if (iteratorIndex >= dictionaryChunks.get(outerIndex).size()) {
+      iteratorIndex = 0;
+      outerIndex++;
+    }
+    byte[] value = dictionaryChunks.get(outerIndex).get(iteratorIndex);
+    currentIndex++;
+    iteratorIndex++;
+    return value;
+  }
+
+  /**
+   * Removes from the underlying collection the last element returned
+   * by this iterator (optional operation).  This method can be called
+   * only once per call to {@link #next}.  The behavior of an iterator
+   * is unspecified if the underlying collection is modified while the
+   * iteration is in progress in any way other than by calling this
+   * method.
+   *
+   * @throws UnsupportedOperationException if the {@code remove}
+   *                                       operation is not supported by this iterator
+   * @throws IllegalStateException         if the {@code next} method has not
+   *                                       yet been called, or the {@code remove} method has already
+   *                                       been called after the last call to the {@code next}
+   *                                       method
+   * @implSpec The default implementation throws an instance of
+   * {@link UnsupportedOperationException} and performs no other action.
+   */
+  @Override public void remove() {
+    throw new UnsupportedOperationException("Remove operation not supported");
+  }
+
+  /**
+   * returns the total element size in List<List<byte[]>>
+   *
+   * @return
+   */
+  public int getSize() {
+    return size;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
new file mode 100644
index 0000000..c9b08dc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
+import org.apache.carbondata.core.carbon.ColumnIdentifier;
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+
+/**
+ * dictionary column identifier which includes table identifier and column identifier
+ */
+public class DictionaryColumnUniqueIdentifier {
+
+  /**
+   * table fully qualified name
+   */
+  private CarbonTableIdentifier carbonTableIdentifier;
+
+  /**
+   * unique column id
+   */
+  private ColumnIdentifier columnIdentifier;
+
+  private DataType dataType;
+
+  /**
+   * Will be used in case of reverse dictionary cache which will be used
+   * in case of data loading.
+   *
+   * @param carbonTableIdentifier
+   * @param columnIdentifier
+   */
+  public DictionaryColumnUniqueIdentifier(CarbonTableIdentifier carbonTableIdentifier,
+      ColumnIdentifier columnIdentifier) {
+    this.carbonTableIdentifier = carbonTableIdentifier;
+    this.columnIdentifier = columnIdentifier;
+  }
+
+  /**
+   * Will be used in case of forward dictionary cache in case
+   * of query execution.
+   *
+   * @param carbonTableIdentifier
+   * @param columnIdentifier
+   * @param dataType
+   */
+  public DictionaryColumnUniqueIdentifier(CarbonTableIdentifier carbonTableIdentifier,
+      ColumnIdentifier columnIdentifier, DataType dataType) {
+    this(carbonTableIdentifier, columnIdentifier);
+    this.dataType = dataType;
+  }
+
+  public DataType getDataType() {
+    return dataType;
+  }
+
+  /**
+   * @return table identifier
+   */
+  public CarbonTableIdentifier getCarbonTableIdentifier() {
+    return carbonTableIdentifier;
+  }
+
+  /**
+   * @return columnIdentifier
+   */
+  public ColumnIdentifier getColumnIdentifier() {
+    return columnIdentifier;
+  }
+
+  /**
+   * overridden equals method
+   *
+   * @param other
+   * @return
+   */
+  @Override public boolean equals(Object other) {
+    if (this == other) return true;
+    if (other == null || getClass() != other.getClass()) return false;
+    DictionaryColumnUniqueIdentifier that = (DictionaryColumnUniqueIdentifier) other;
+    if (!carbonTableIdentifier.equals(that.carbonTableIdentifier)) return false;
+    return columnIdentifier.equals(that.columnIdentifier);
+
+  }
+
+  /**
+   * overridden hashcode method
+   *
+   * @return
+   */
+  @Override public int hashCode() {
+    int result = carbonTableIdentifier.hashCode();
+    result = 31 * result + columnIdentifier.hashCode();
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryInfo.java
new file mode 100644
index 0000000..e34860a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryInfo.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.List;
+
+import org.apache.carbondata.core.cache.Cacheable;
+
+/**
+ * An interface which holds dictionary information like end offset,
+ * file timestamp for one column
+ */
+public interface DictionaryInfo extends Cacheable, Dictionary {
+
+  /**
+   * This method will increment the access count for a column by 1
+   * whenever a column is getting used in query or incremental data load
+   */
+  void incrementAccessCount();
+
+  /**
+   * This method will update the end offset of file everytime a file is read
+   *
+   * @param offsetTillFileIsRead
+   */
+  void setOffsetTillFileIsRead(long offsetTillFileIsRead);
+
+  /**
+   * This method will update the timestamp of a file if a file is modified
+   * like in case of incremental load
+   *
+   * @param fileTimeStamp
+   */
+  void setFileTimeStamp(long fileTimeStamp);
+
+  /**
+   * This method will add a new dictionary chunk to existing list of dictionary chunks
+   *
+   * @param dictionaryChunk
+   */
+  void addDictionaryChunk(List<byte[]> dictionaryChunk);
+
+  /**
+   * This method will set the sort order index of a dictionary column.
+   * Sort order index if the index of dictionary values after they are sorted.
+   *
+   * @param sortOrderIndex
+   */
+  void setSortOrderIndex(List<Integer> sortOrderIndex);
+
+  /**
+   * This method will set the sort reverse index of a dictionary column.
+   * Sort reverse index is the index of dictionary values before they are sorted.
+   *
+   * @param sortReverseOrderIndex
+   */
+  void setSortReverseOrderIndex(List<Integer> sortReverseOrderIndex);
+
+  /**
+   * dictionary metadata file length which will be set whenever we reload dictionary
+   * data from disk
+   *
+   * @param dictionaryMetaFileLength length of dictionary metadata file
+   */
+  void setDictionaryMetaFileLength(long dictionaryMetaFileLength);
+
+  /**
+   * Dictionary meta file offset which will be read to check whether length of dictionary
+   * meta file has been modified
+   *
+   * @return
+   */
+  long getDictionaryMetaFileLength();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
new file mode 100644
index 0000000..bddab56
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionary.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * This class will be used for dictionary key and value look up
+ */
+public class ForwardDictionary implements Dictionary {
+
+  /**
+   * Object which will hold the information related to this dictionary column
+   */
+  private ColumnDictionaryInfo columnDictionaryInfo;
+
+  /**
+   * @param columnDictionaryInfo
+   */
+  public ForwardDictionary(ColumnDictionaryInfo columnDictionaryInfo) {
+    this.columnDictionaryInfo = columnDictionaryInfo;
+  }
+
+  /**
+   * This method will find and return the surrogate key for a given dictionary value
+   * Applicable scenario:
+   * 1. Incremental data load : Dictionary will not be generated for existing values. For
+   * that values have to be looked up in the existing dictionary cache.
+   * 2. Filter scenarios where from value surrogate key has to be found.
+   *
+   * @param value dictionary value
+   * @return if found returns key else 0
+   */
+  @Override public int getSurrogateKey(String value) {
+    return columnDictionaryInfo.getSurrogateKey(value);
+  }
+
+  /**
+   * This method will find and return the surrogate key for a given dictionary value
+   * Applicable scenario:
+   * 1. Incremental data load : Dictionary will not be generated for existing values. For
+   * that values have to be looked up in the existing dictionary cache.
+   * 2. Filter scenarios where from value surrogate key has to be found.
+   *
+   * @param value dictionary value as byte array
+   * @return if found returns key else 0
+   */
+  @Override public int getSurrogateKey(byte[] value) {
+    return columnDictionaryInfo.getSurrogateKey(value);
+  }
+
+  /**
+   * This method will find and return the dictionary value for a given surrogate key.
+   * Applicable scenarios:
+   * 1. Query final result preparation : While convert the final result which will
+   * be surrogate key back to original dictionary values this method will be used
+   *
+   * @param surrogateKey a unique ID for a dictionary value
+   * @return value if found else null
+   */
+  @Override public String getDictionaryValueForKey(int surrogateKey) {
+    return columnDictionaryInfo.getDictionaryValueForKey(surrogateKey);
+  }
+
+  /**
+   * This method will find and return the sort index for a given dictionary id.
+   * Applicable scenarios:
+   * 1. Used in case of order by queries when data sorting is required
+   *
+   * @param surrogateKey a unique ID for a dictionary value
+   * @return if found returns key else 0
+   */
+  @Override public int getSortedIndex(int surrogateKey) {
+    return columnDictionaryInfo.getSortedIndex(surrogateKey);
+  }
+
+  /**
+   * This method will find and return the dictionary value from sorted index.
+   * Applicable scenarios:
+   * 1. Query final result preparation in case of order by queries:
+   * While convert the final result which will
+   * be surrogate key back to original dictionary values this method will be used
+   *
+   * @param sortedIndex sort index of dictionary value
+   * @return value if found else null
+   */
+  @Override public String getDictionaryValueFromSortedIndex(int sortedIndex) {
+    return columnDictionaryInfo.getDictionaryValueFromSortedIndex(sortedIndex);
+  }
+
+  /**
+   * The method return the dictionary chunks wrapper of a column
+   * The wrapper wraps the list<list<bye[]>> and provide the iterator to retrieve the chunks
+   * members.
+   * Applications Scenario:
+   * For preparing the column Sort info while writing the sort index file.
+   *
+   * @return
+   */
+  @Override public DictionaryChunksWrapper getDictionaryChunks() {
+    return columnDictionaryInfo.getDictionaryChunks();
+  }
+
+  /**
+   * This method will release the objects and set default value for primitive types
+   */
+  @Override public void clear() {
+    if (null != columnDictionaryInfo) {
+      columnDictionaryInfo.clear();
+      columnDictionaryInfo = null;
+    }
+  }
+
+  /**
+   * This method will read the surrogates based on search range.
+   *
+   * @param surrogates
+   */
+  public void getSurrogateKeyByIncrementalSearch(List<String> evaluateResultList,
+      List<Integer> surrogates) {
+    List<byte[]> byteValuesOfFilterMembers = new ArrayList<byte[]>(evaluateResultList.size());
+    byte[] keyData = null;
+    for (int i = 0; i < evaluateResultList.size(); i++) {
+      keyData = evaluateResultList.get(i)
+          .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+      byteValuesOfFilterMembers.add(keyData);
+    }
+
+    columnDictionaryInfo
+        .getIncrementalSurrogateKeyFromDictionary(byteValuesOfFilterMembers, surrogates);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
new file mode 100644
index 0000000..d4f6adc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCache.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.CarbonLRUCache;
+import org.apache.carbondata.core.util.CarbonUtilException;
+
+/**
+ * This class implements methods to create dictionary cache which will hold
+ * dictionary chunks for look up of surrogate keys and values
+ */
+public class ForwardDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
+                                    V extends Dictionary>
+    extends AbstractDictionaryCache<K, V> {
+
+  /**
+   * Attribute for Carbon LOGGER
+   */
+  private static final LogService LOGGER =
+      LogServiceFactory.getLogService(ForwardDictionaryCache.class.getName());
+
+  /**
+   * @param carbonStorePath
+   * @param carbonLRUCache
+   */
+  public ForwardDictionaryCache(String carbonStorePath, CarbonLRUCache carbonLRUCache) {
+    super(carbonStorePath, carbonLRUCache);
+  }
+
+  /**
+   * This method will get the value for the given key. If value does not exist
+   * for the given key, it will check and load the value.
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   * @return dictionary
+   * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+   */
+  @Override public Dictionary get(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
+      throws CarbonUtilException {
+    return getDictionary(dictionaryColumnUniqueIdentifier);
+  }
+
+  /**
+   * This method will return a list of values for the given list of keys.
+   * For each key, this method will check and load the data if required.
+   *
+   * @param dictionaryColumnUniqueIdentifiers unique identifier which contains dbName,
+   *                                          tableName and columnIdentifier
+   * @return list of dictionary
+   * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+   */
+  @Override public List<Dictionary> getAll(
+      List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers)
+      throws CarbonUtilException {
+    boolean exceptionOccurredInDictionaryLoading = false;
+    String exceptionMessage = "";
+    List<Dictionary> forwardDictionaryObjectList =
+        new ArrayList<Dictionary>(dictionaryColumnUniqueIdentifiers.size());
+    List<Future<Dictionary>> taskSubmitList =
+        new ArrayList<>(dictionaryColumnUniqueIdentifiers.size());
+    ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
+    for (final DictionaryColumnUniqueIdentifier uniqueIdent : dictionaryColumnUniqueIdentifiers) {
+      taskSubmitList.add(executorService.submit(new Callable<Dictionary>() {
+        @Override public Dictionary call() throws CarbonUtilException {
+          Dictionary dictionary = getDictionary(uniqueIdent);
+          return dictionary;
+        }
+      }));
+    }
+    try {
+      executorService.shutdown();
+      executorService.awaitTermination(2, TimeUnit.HOURS);
+    } catch (InterruptedException e) {
+      LOGGER.error("Error loading the dictionary: " + e.getMessage());
+    }
+    for (int i = 0; i < taskSubmitList.size(); i++) {
+      try {
+        Dictionary columnDictionary = taskSubmitList.get(i).get();
+        forwardDictionaryObjectList.add(columnDictionary);
+      } catch (Throwable e) {
+        exceptionOccurredInDictionaryLoading = true;
+        exceptionMessage = e.getMessage();
+      }
+    }
+    if (exceptionOccurredInDictionaryLoading) {
+      clearDictionary(forwardDictionaryObjectList);
+      LOGGER.error(exceptionMessage);
+      throw new CarbonUtilException(exceptionMessage);
+    }
+    return forwardDictionaryObjectList;
+  }
+
+  /**
+   * This method will return the value for the given key. It will not check and load
+   * the data for the given key
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   * @return
+   */
+  @Override public Dictionary getIfPresent(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+    Dictionary forwardDictionary = null;
+    ColumnDictionaryInfo columnDictionaryInfo = (ColumnDictionaryInfo) carbonLRUCache.get(
+        getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+            CacheType.FORWARD_DICTIONARY));
+    if (null != columnDictionaryInfo) {
+      forwardDictionary = new ForwardDictionary(columnDictionaryInfo);
+      incrementDictionaryAccessCount(columnDictionaryInfo);
+    }
+    return forwardDictionary;
+  }
+
+  /**
+   * This method will remove the cache for a given key
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   */
+  @Override public void invalidate(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+    carbonLRUCache.remove(
+        getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+            CacheType.FORWARD_DICTIONARY));
+  }
+
+  /**
+   * This method will get the value for the given key. If value does not exist
+   * for the given key, it will check and load the value.
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   * @return dictionary
+   * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+   */
+  private Dictionary getDictionary(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
+      throws CarbonUtilException {
+    Dictionary forwardDictionary = null;
+    // create column dictionary info object only if dictionary and its
+    // metadata file exists for a given column identifier
+    if (!isFileExistsForGivenColumn(dictionaryColumnUniqueIdentifier)) {
+      throw new CarbonUtilException(
+          "Either dictionary or its metadata does not exist for column identifier :: "
+              + dictionaryColumnUniqueIdentifier.getColumnIdentifier());
+    }
+    String columnIdentifier = dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId();
+    ColumnDictionaryInfo columnDictionaryInfo =
+        getColumnDictionaryInfo(dictionaryColumnUniqueIdentifier, columnIdentifier);
+    // load sort index file in case of forward dictionary
+    checkAndLoadDictionaryData(dictionaryColumnUniqueIdentifier, columnDictionaryInfo,
+        getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+            CacheType.FORWARD_DICTIONARY), true);
+    forwardDictionary = new ForwardDictionary(columnDictionaryInfo);
+    return forwardDictionary;
+  }
+
+  /**
+   * This method will check and create columnDictionaryInfo object for the given column
+   *
+   * @param dictionaryColumnUniqueIdentifier
+   * @param columnIdentifier
+   * @return
+   */
+  private ColumnDictionaryInfo getColumnDictionaryInfo(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String columnIdentifier) {
+    ColumnDictionaryInfo columnDictionaryInfo = (ColumnDictionaryInfo) carbonLRUCache
+        .get(getLruCacheKey(columnIdentifier, CacheType.FORWARD_DICTIONARY));
+    if (null == columnDictionaryInfo) {
+      synchronized (dictionaryColumnUniqueIdentifier) {
+        columnDictionaryInfo = (ColumnDictionaryInfo) carbonLRUCache
+            .get(getLruCacheKey(columnIdentifier, CacheType.FORWARD_DICTIONARY));
+        if (null == columnDictionaryInfo) {
+          columnDictionaryInfo =
+              new ColumnDictionaryInfo(dictionaryColumnUniqueIdentifier.getDataType());
+        }
+      }
+    }
+    return columnDictionaryInfo;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
new file mode 100644
index 0000000..1f8a3ff
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionary.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+/**
+ * This class will be used for dictionary key and value look up
+ */
+public class ReverseDictionary implements Dictionary {
+
+  /**
+   * Object which will hold the information related to this dictionary column
+   */
+  private ColumnReverseDictionaryInfo columnReverseDictionaryInfo;
+
+  /**
+   * @param columnReverseDictionaryInfo
+   */
+  public ReverseDictionary(ColumnReverseDictionaryInfo columnReverseDictionaryInfo) {
+    this.columnReverseDictionaryInfo = columnReverseDictionaryInfo;
+  }
+
+  /**
+   * This method will find and return the surrogate key for a given dictionary value
+   * Applicable scenario:
+   * 1. Incremental data load : Dictionary will not be generated for existing values. For
+   * that values have to be looked up in the existing dictionary cache.
+   * 2. Filter scenarios where from value surrogate key has to be found.
+   *
+   * @param value dictionary value
+   * @return if found returns key else 0
+   */
+  @Override public int getSurrogateKey(String value) {
+    return columnReverseDictionaryInfo.getSurrogateKey(value);
+  }
+
+  /**
+   * This method will find and return the surrogate key for a given dictionary value
+   * Applicable scenario:
+   * 1. Incremental data load : Dictionary will not be generated for existing values. For
+   * that values have to be looked up in the existing dictionary cache.
+   * 2. Filter scenarios where from value surrogate key has to be found.
+   *
+   * @param value dictionary value as byte array
+   * @return if found returns key else 0
+   */
+  @Override public int getSurrogateKey(byte[] value) {
+    return columnReverseDictionaryInfo.getSurrogateKey(value);
+  }
+
+  /**
+   * This method will find and return the dictionary value for a given surrogate key.
+   * Applicable scenarios:
+   * 1. Query final result preparation : While convert the final result which will
+   * be surrogate key back to original dictionary values this method will be used
+   *
+   * @param surrogateKey a unique ID for a dictionary value
+   * @return value if found else null
+   */
+  @Override public String getDictionaryValueForKey(int surrogateKey) {
+    return columnReverseDictionaryInfo.getDictionaryValueForKey(surrogateKey);
+  }
+
+  /**
+   * This method will find and return the sort index for a given dictionary id.
+   * Applicable scenarios:
+   * 1. Used in case of order by queries when data sorting is required
+   *
+   * @param surrogateKey a unique ID for a dictionary value
+   * @return if found returns key else 0
+   */
+  @Override public int getSortedIndex(int surrogateKey) {
+    return columnReverseDictionaryInfo.getSortedIndex(surrogateKey);
+  }
+
+  /**
+   * This method will find and return the dictionary value from sorted index.
+   * Applicable scenarios:
+   * 1. Query final result preparation in case of order by queries:
+   * While convert the final result which will
+   * be surrogate key back to original dictionary values this method will be used
+   *
+   * @param sortedIndex sort index of dictionary value
+   * @return value if found else null
+   */
+  @Override public String getDictionaryValueFromSortedIndex(int sortedIndex) {
+    return columnReverseDictionaryInfo.getDictionaryValueFromSortedIndex(sortedIndex);
+  }
+
+  /**
+   * The method return the dictionary chunks wrapper of a column
+   * The wrapper wraps the list<list<bye[]>> and provide the iterator to retrieve the chunks
+   * members.
+   * Applications Scenario:
+   * For preparing the column Sort info while writing the sort index file.
+   *
+   * @return
+   */
+  @Override public DictionaryChunksWrapper getDictionaryChunks() {
+    return columnReverseDictionaryInfo.getDictionaryChunks();
+  }
+
+  /**
+   * This method will release the objects and set default value for primitive types
+   */
+  @Override public void clear() {
+    if (null != columnReverseDictionaryInfo) {
+      columnReverseDictionaryInfo.clear();
+      columnReverseDictionaryInfo = null;
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
new file mode 100644
index 0000000..aa05570
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.cache.dictionary;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.CarbonLRUCache;
+import org.apache.carbondata.core.util.CarbonUtilException;
+
+/**
+ * This class implements methods to create dictionary cache which will hold
+ * dictionary chunks for look up of surrogate keys and values
+ */
+public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
+    V extends Dictionary>
+    extends AbstractDictionaryCache<K, V> {
+
+  /**
+   * Attribute for Carbon LOGGER
+   */
+  private static final LogService LOGGER =
+      LogServiceFactory.getLogService(ForwardDictionaryCache.class.getName());
+
+  /**
+   * @param carbonStorePath
+   * @param carbonLRUCache
+   */
+  public ReverseDictionaryCache(String carbonStorePath, CarbonLRUCache carbonLRUCache) {
+    super(carbonStorePath, carbonLRUCache);
+  }
+
+  /**
+   * This method will get the value for the given key. If value does not exist
+   * for the given key, it will check and load the value.
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   * @return dictionary
+   * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+   */
+  @Override public Dictionary get(DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
+      throws CarbonUtilException {
+    return getDictionary(dictionaryColumnUniqueIdentifier);
+  }
+
+  /**
+   * This method will return a list of values for the given list of keys.
+   * For each key, this method will check and load the data if required.
+   *
+   * @param dictionaryColumnUniqueIdentifiers unique identifier which contains dbName,
+   *                                          tableName and columnIdentifier
+   * @return list of dictionary
+   * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+   */
+  @Override public List<Dictionary> getAll(
+      List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers)
+      throws CarbonUtilException {
+    boolean exceptionOccurredInDictionaryLoading = false;
+    String exceptionMessage = "";
+    List<Dictionary> reverseDictionaryObjectList =
+        new ArrayList<Dictionary>(dictionaryColumnUniqueIdentifiers.size());
+    List<Future<Dictionary>> taskSubmitList =
+        new ArrayList<>(dictionaryColumnUniqueIdentifiers.size());
+    ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
+    for (final DictionaryColumnUniqueIdentifier uniqueIdent : dictionaryColumnUniqueIdentifiers) {
+      taskSubmitList.add(executorService.submit(new Callable<Dictionary>() {
+        @Override public Dictionary call() throws CarbonUtilException {
+          Dictionary dictionary = getDictionary(uniqueIdent);
+          return dictionary;
+        }
+      }));
+    }
+    try {
+      executorService.shutdown();
+      executorService.awaitTermination(2, TimeUnit.HOURS);
+    } catch (InterruptedException e) {
+      LOGGER.error("Error loading the dictionary: " + e.getMessage());
+    }
+    for (int i = 0; i < taskSubmitList.size(); i++) {
+      try {
+        Dictionary columnDictionary = taskSubmitList.get(i).get();
+        reverseDictionaryObjectList.add(columnDictionary);
+      } catch (Throwable e) {
+        exceptionOccurredInDictionaryLoading = true;
+        exceptionMessage = e.getMessage();
+      }
+    }
+    if (exceptionOccurredInDictionaryLoading) {
+      clearDictionary(reverseDictionaryObjectList);
+      LOGGER.error(exceptionMessage);
+      throw new CarbonUtilException(exceptionMessage);
+    }
+    return reverseDictionaryObjectList;
+  }
+
+  /**
+   * This method will return the value for the given key. It will not check and load
+   * the data for the given key
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   * @return
+   */
+  @Override public Dictionary getIfPresent(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+    Dictionary reverseDictionary = null;
+    ColumnReverseDictionaryInfo columnReverseDictionaryInfo =
+        (ColumnReverseDictionaryInfo) carbonLRUCache.get(
+            getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+                CacheType.REVERSE_DICTIONARY));
+    if (null != columnReverseDictionaryInfo) {
+      reverseDictionary = new ReverseDictionary(columnReverseDictionaryInfo);
+      incrementDictionaryAccessCount(columnReverseDictionaryInfo);
+    }
+    return reverseDictionary;
+  }
+
+  /**
+   * This method will remove the cache for a given key
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   */
+  @Override public void invalidate(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier) {
+    carbonLRUCache.remove(
+        getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+            CacheType.REVERSE_DICTIONARY));
+  }
+
+  /**
+   * This method will get the value for the given key. If value does not exist
+   * for the given key, it will check and load the value.
+   *
+   * @param dictionaryColumnUniqueIdentifier unique identifier which contains dbName,
+   *                                         tableName and columnIdentifier
+   * @return dictionary
+   * @throws CarbonUtilException in case memory is not sufficient to load dictionary into memory
+   */
+  private Dictionary getDictionary(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier)
+      throws CarbonUtilException {
+    Dictionary reverseDictionary = null;
+    // create column dictionary info object only if dictionary and its
+    // metadata file exists for a given column identifier
+    if (!isFileExistsForGivenColumn(dictionaryColumnUniqueIdentifier)) {
+      throw new CarbonUtilException(
+          "Either dictionary or its metadata does not exist for column identifier :: "
+              + dictionaryColumnUniqueIdentifier.getColumnIdentifier());
+    }
+    String columnIdentifier = dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId();
+    ColumnReverseDictionaryInfo columnReverseDictionaryInfo =
+        getColumnReverseDictionaryInfo(dictionaryColumnUniqueIdentifier, columnIdentifier);
+    // do not load sort index file for reverse dictionary
+    checkAndLoadDictionaryData(dictionaryColumnUniqueIdentifier, columnReverseDictionaryInfo,
+        getLruCacheKey(dictionaryColumnUniqueIdentifier.getColumnIdentifier().getColumnId(),
+            CacheType.REVERSE_DICTIONARY), false);
+    reverseDictionary = new ReverseDictionary(columnReverseDictionaryInfo);
+    return reverseDictionary;
+  }
+
+  /**
+   * This method will check and create columnReverseDictionaryInfo object for the given column
+   *
+   * @param dictionaryColumnUniqueIdentifier
+   * @param columnIdentifier
+   * @return
+   */
+  private ColumnReverseDictionaryInfo getColumnReverseDictionaryInfo(
+      DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier, String columnIdentifier) {
+    ColumnReverseDictionaryInfo columnReverseDictionaryInfo =
+        (ColumnReverseDictionaryInfo) carbonLRUCache
+            .get(getLruCacheKey(columnIdentifier, CacheType.REVERSE_DICTIONARY));
+    if (null == columnReverseDictionaryInfo) {
+      synchronized (dictionaryColumnUniqueIdentifier) {
+        columnReverseDictionaryInfo = (ColumnReverseDictionaryInfo) carbonLRUCache
+            .get(getLruCacheKey(columnIdentifier, CacheType.REVERSE_DICTIONARY));
+        if (null == columnReverseDictionaryInfo) {
+          columnReverseDictionaryInfo = new ColumnReverseDictionaryInfo();
+        }
+      }
+    }
+    return columnReverseDictionaryInfo;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifier.java
new file mode 100644
index 0000000..c8f603a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifier.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+
+/**
+ * identifier which will have store path and carbon table identifier
+ */
+public class AbsoluteTableIdentifier implements Serializable {
+
+  /**
+   * serializable version
+   */
+  private static final long serialVersionUID = 4695047103484427506L;
+
+  /**
+   * path of the store
+   */
+  private String storePath;
+
+  /**
+   * carbon table identifier which will have table name and table database
+   * name
+   */
+  private CarbonTableIdentifier carbonTableIdentifier;
+
+  public AbsoluteTableIdentifier(String storePath, CarbonTableIdentifier carbonTableIdentifier) {
+    //TODO this should be moved to common place where path handling will be handled
+    this.storePath = FileFactory.getUpdatedFilePath(storePath);
+    this.carbonTableIdentifier = carbonTableIdentifier;
+  }
+
+  /**
+   * @return the storePath
+   */
+  public String getStorePath() {
+    return storePath;
+  }
+
+  /**
+   * @return the carbonTableIdentifier
+   */
+  public CarbonTableIdentifier getCarbonTableIdentifier() {
+    return carbonTableIdentifier;
+  }
+
+  /**
+   * to get the hash code
+   */
+  @Override public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result =
+        prime * result + ((carbonTableIdentifier == null) ? 0 : carbonTableIdentifier.hashCode());
+    result = prime * result + ((storePath == null) ? 0 : storePath.hashCode());
+    return result;
+  }
+
+  /**
+   * to check this class is equal to
+   * other object passed
+   *
+   * @param obj other object
+   */
+  @Override public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (!(obj instanceof AbsoluteTableIdentifier)) {
+      return false;
+    }
+    AbsoluteTableIdentifier other = (AbsoluteTableIdentifier) obj;
+    if (carbonTableIdentifier == null) {
+      if (other.carbonTableIdentifier != null) {
+        return false;
+      }
+    } else if (!carbonTableIdentifier.equals(other.carbonTableIdentifier)) {
+      return false;
+    }
+    if (storePath == null) {
+      if (other.storePath != null) {
+        return false;
+      }
+    } else if (!storePath.equals(other.storePath)) {
+      return false;
+    }
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
new file mode 100644
index 0000000..7cfefc9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonDataLoadSchema.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
+
+/**
+ * Wrapper Data Load Schema object which will be used to
+ * support relation while data loading
+ */
+public class CarbonDataLoadSchema implements Serializable {
+
+  /**
+   * default serializer
+   */
+  private static final long serialVersionUID = 1L;
+
+  /**
+   * CarbonTable info
+   */
+  private CarbonTable carbonTable;
+
+  /**
+   * dimension table and relation info
+   */
+  private List<DimensionRelation> dimensionRelationList;
+
+  /**
+   * CarbonDataLoadSchema constructor which takes CarbonTable
+   *
+   * @param carbonTable
+   */
+  public CarbonDataLoadSchema(CarbonTable carbonTable) {
+    this.carbonTable = carbonTable;
+    this.dimensionRelationList = new ArrayList<DimensionRelation>();
+  }
+
+  /**
+   * get dimension relation list
+   *
+   * @return dimensionRelationList
+   */
+  public List<DimensionRelation> getDimensionRelationList() {
+    return dimensionRelationList;
+  }
+
+  /**
+   * set dimensionrelation list
+   *
+   * @param dimensionRelationList
+   */
+  public void setDimensionRelationList(List<DimensionRelation> dimensionRelationList) {
+    this.dimensionRelationList = dimensionRelationList;
+  }
+
+  /**
+   * get carbontable
+   *
+   * @return carbonTable
+   */
+  public CarbonTable getCarbonTable() {
+    return carbonTable;
+  }
+
+  /**
+   * Dimension Relation object which will be filled from
+   * Load DML Command to support normalized table data load
+   */
+  public static class DimensionRelation implements Serializable {
+    /**
+     * default serializer
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * dimension tableName
+     */
+    private String tableName;
+
+    /**
+     * dimensionSource csv path
+     */
+    private String dimensionSource;
+
+    /**
+     * relation with fact and dimension table
+     */
+    private Relation relation;
+
+    /**
+     * Columns to selected from dimension table.
+     * Hierarchy in-memory table should be prepared
+     * based on selected columns
+     */
+    private List<String> columns;
+
+    /**
+     * constructor
+     *
+     * @param tableName       - dimension table name
+     * @param dimensionSource - source file path
+     * @param relation        - fact foreign key with dimension primary key mapping
+     * @param columns         - list of columns to be used from this dimension table
+     */
+    public DimensionRelation(String tableName, String dimensionSource, Relation relation,
+        List<String> columns) {
+      this.tableName = tableName;
+      this.dimensionSource = dimensionSource;
+      this.relation = relation;
+      this.columns = columns;
+    }
+
+    /**
+     * @return tableName
+     */
+    public String getTableName() {
+      return tableName;
+    }
+
+    /**
+     * @return dimensionSource
+     */
+    public String getDimensionSource() {
+      return dimensionSource;
+    }
+
+    /**
+     * @return relation
+     */
+    public Relation getRelation() {
+      return relation;
+    }
+
+    /**
+     * @return columns
+     */
+    public List<String> getColumns() {
+      return columns;
+    }
+  }
+
+  /**
+   * Relation class to specify fact foreignkey column with
+   * dimension primary key column
+   */
+  public static class Relation implements Serializable {
+    /**
+     * default serializer
+     */
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Fact foreign key column
+     */
+    private String factForeignKeyColumn;
+
+    /**
+     * dimension primary key column
+     */
+    private String dimensionPrimaryKeyColumn;
+
+    /**
+     * constructor
+     *
+     * @param factForeignKeyColumn      - Fact Table Foreign key
+     * @param dimensionPrimaryKeyColumn - Dimension Table primary key
+     */
+    public Relation(String factForeignKeyColumn, String dimensionPrimaryKeyColumn) {
+      this.factForeignKeyColumn = factForeignKeyColumn;
+      this.dimensionPrimaryKeyColumn = dimensionPrimaryKeyColumn;
+    }
+
+    /**
+     * @return factForeignKeyColumn
+     */
+    public String getFactForeignKeyColumn() {
+      return factForeignKeyColumn;
+    }
+
+    /**
+     * @return dimensionPrimaryKeyColumn
+     */
+    public String getDimensionPrimaryKeyColumn() {
+      return dimensionPrimaryKeyColumn;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/CarbonTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/CarbonTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonTableIdentifier.java
new file mode 100644
index 0000000..bb8a816
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/CarbonTableIdentifier.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.carbon;
+
+import java.io.Serializable;
+
+/**
+ * Identifier class which will hold the table qualified name
+ */
+public class CarbonTableIdentifier implements Serializable {
+
+  /**
+   * database name
+   */
+  private String databaseName;
+
+  /**
+   * table name
+   */
+  private String tableName;
+
+  /**
+   * table id
+   */
+  private String tableId;
+
+  /**
+   * constructor
+   */
+  public CarbonTableIdentifier(String databaseName, String tableName, String tableId) {
+    this.databaseName = databaseName;
+    this.tableName = tableName;
+    this.tableId = tableId;
+  }
+
+  /**
+   * return database name
+   */
+  public String getDatabaseName() {
+    return databaseName;
+  }
+
+  /**
+   * return table name
+   */
+  public String getTableName() {
+    return tableName;
+  }
+
+  /**
+   * @return tableId
+   */
+  public String getTableId() {
+    return tableId;
+  }
+
+  /**
+   * @return table unique name
+   */
+  public String getTableUniqueName() {
+    return databaseName + '_' + tableName;
+  }
+
+  @Override public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((databaseName == null) ? 0 : databaseName.hashCode());
+    result = prime * result + ((tableId == null) ? 0 : tableId.hashCode());
+    result = prime * result + ((tableName == null) ? 0 : tableName.hashCode());
+    return result;
+  }
+
+  @Override public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (getClass() != obj.getClass()) {
+      return false;
+    }
+    CarbonTableIdentifier other = (CarbonTableIdentifier) obj;
+    if (databaseName == null) {
+      if (other.databaseName != null) {
+        return false;
+      }
+    } else if (!databaseName.equals(other.databaseName)) {
+      return false;
+    }
+    if (tableId == null) {
+      if (other.tableId != null) {
+        return false;
+      }
+    } else if (!tableId.equals(other.tableId)) {
+      return false;
+    }
+    if (tableName == null) {
+      if (other.tableName != null) {
+        return false;
+      }
+    } else if (!tableName.equals(other.tableName)) {
+      return false;
+    }
+    return true;
+  }
+
+  /*
+ * @return table unidque name
+ */
+  @Override public String toString() {
+    return databaseName + '_' + tableName;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/ColumnIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/ColumnIdentifier.java b/core/src/main/java/org/apache/carbondata/core/carbon/ColumnIdentifier.java
new file mode 100644
index 0000000..9dda92c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/ColumnIdentifier.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon;
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+
+/**
+ * Column unique identifier
+ */
+public class ColumnIdentifier implements Serializable {
+
+  /**
+   *
+   */
+  private static final long serialVersionUID = 1L;
+
+  /**
+   * column id
+   */
+  private String columnId;
+
+  /**
+   * column properties
+   */
+  private Map<String, String> columnProperties;
+
+  private DataType dataType;
+
+  /**
+   * @param columnId
+   * @param columnProperties
+   */
+  public ColumnIdentifier(String columnId, Map<String, String> columnProperties,
+      DataType dataType) {
+    this.columnId = columnId;
+    this.columnProperties = columnProperties;
+    this.dataType = dataType;
+  }
+
+  /**
+   * @return columnId
+   */
+  public String getColumnId() {
+    return columnId;
+  }
+
+  /**
+   * @param columnProperty
+   * @return
+   */
+  public String getColumnProperty(String columnProperty) {
+    if (null != columnProperties) {
+      return columnProperties.get(columnProperty);
+    }
+    return null;
+  }
+
+  public DataType getDataType() {
+    return this.dataType;
+  }
+
+  @Override public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((columnId == null) ? 0 : columnId.hashCode());
+    return result;
+  }
+
+  @Override public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (getClass() != obj.getClass()) {
+      return false;
+    }
+    ColumnIdentifier other = (ColumnIdentifier) obj;
+    if (columnId == null) {
+      if (other.columnId != null) {
+        return false;
+      }
+    } else if (!columnId.equals(other.columnId)) {
+      return false;
+    }
+    return true;
+  }
+
+  @Override public String toString() {
+    return "ColumnIdentifier [columnId=" + columnId + "]";
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BTreeBuilderInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BTreeBuilderInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BTreeBuilderInfo.java
new file mode 100644
index 0000000..9c090da
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BTreeBuilderInfo.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
+
+/**
+ * below class holds the meta data requires to build the blocks
+ */
+public class BTreeBuilderInfo {
+
+  /**
+   * holds all the information about data
+   * file meta data
+   */
+  private List<DataFileFooter> footerList;
+
+  /**
+   * size of the each column value size
+   * this will be useful for reading
+   */
+  private int[] dimensionColumnValueSize;
+
+  public BTreeBuilderInfo(List<DataFileFooter> footerList,
+      int[] dimensionColumnValueSize) {
+    this.dimensionColumnValueSize = dimensionColumnValueSize;
+    this.footerList = footerList;
+  }
+
+  /**
+   * @return the eachDimensionBlockSize
+   */
+  public int[] getDimensionColumnValueSize() {
+    return dimensionColumnValueSize;
+  }
+
+  /**
+   * @return the footerList
+   */
+  public List<DataFileFooter> getFooterList() {
+    return footerList;
+  }
+}


Mime
View raw message