carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [19/52] [partial] incubator-carbondata git commit: move core package
Date Mon, 16 Jan 2017 14:52:56 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
deleted file mode 100644
index 3071a98..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
+++ /dev/null
@@ -1,984 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.core.updatestatus;
-
-import java.io.*;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.path.CarbonStorePath;
-import org.apache.carbondata.core.carbon.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
-import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
-import org.apache.carbondata.core.update.CarbonUpdateUtil;
-import org.apache.carbondata.core.update.SegmentUpdateDetails;
-import org.apache.carbondata.core.update.TupleIdEnum;
-import org.apache.carbondata.core.update.UpdateVO;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl;
-import org.apache.carbondata.fileoperations.FileWriteOperation;
-import org.apache.carbondata.locks.CarbonLockFactory;
-import org.apache.carbondata.locks.ICarbonLock;
-import org.apache.carbondata.locks.LockUsage;
-
-import com.google.gson.Gson;
-
-/**
- * Manages Segment & block status of carbon table for Delete operation
- */
-public class SegmentUpdateStatusManager {
-
-  /**
-   * logger
-   */
-  private static final LogService LOG =
-      LogServiceFactory.getLogService(SegmentUpdateStatusManager.class.getName());
-
-  private AbsoluteTableIdentifier absoluteTableIdentifier;
-  private LoadMetadataDetails[] segmentDetails;
-  private SegmentUpdateDetails[] updateDetails;
-  private CarbonTablePath carbonTablePath;
-  private Map<String, SegmentUpdateDetails> blockAndDetailsMap;
-
-  /**
-   * @param absoluteTableIdentifier
-   */
-  public SegmentUpdateStatusManager(AbsoluteTableIdentifier absoluteTableIdentifier) {
-    this.absoluteTableIdentifier = absoluteTableIdentifier;
-    carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-        absoluteTableIdentifier.getCarbonTableIdentifier());
-    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
-    // current it is used only for read function scenarios, as file update always requires to work
-    // on latest file status.
-    segmentDetails =
-        segmentStatusManager.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
-    updateDetails = readLoadMetadata();
-    populateMap();
-  }
-
-  /**
-   * populate the block and its details in a map.
-   */
-  private void populateMap() {
-    blockAndDetailsMap = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-    for(SegmentUpdateDetails blockDetails : updateDetails) {
-
-      String blockIdentifier = CarbonUpdateUtil
-          .getSegmentBlockNameKey(blockDetails.getSegmentName(), blockDetails.getActualBlockName());
-
-      blockAndDetailsMap.put(blockIdentifier, blockDetails);
-
-    }
-
-  }
-
-  /**
-   *
-   * @param segID
-   * @param actualBlockName
-   * @return null if block is not present in segment update status.
-   */
-  public SegmentUpdateDetails getDetailsForABlock(String segID, String actualBlockName) {
-
-    String blockIdentifier = CarbonUpdateUtil
-        .getSegmentBlockNameKey(segID, actualBlockName);
-
-    return blockAndDetailsMap.get(blockIdentifier);
-
-  }
-
-  /**
-   *
-   * @param key will be like (segid/blockname)  0/0-0-5464654654654
-   * @return
-   */
-  public SegmentUpdateDetails getDetailsForABlock(String key) {
-
-    return blockAndDetailsMap.get(key);
-
-  }
-
-
-
-  /**
-   * Returns the LoadMetadata Details
-   * @return
-   */
-  public LoadMetadataDetails[] getLoadMetadataDetails() {
-    return segmentDetails;
-  }
-
-  /**
-   *
-   * @param loadMetadataDetails
-   */
-  public void setLoadMetadataDetails(LoadMetadataDetails[] loadMetadataDetails) {
-    this.segmentDetails = loadMetadataDetails;
-  }
-
-  /**
-   * Returns the UpdateStatus Details.
-   * @return
-   */
-  public SegmentUpdateDetails[] getUpdateStatusDetails() {
-    return updateDetails;
-  }
-
-  /**
-   *
-   * @param segmentUpdateDetails
-   */
-  public void setUpdateStatusDetails(SegmentUpdateDetails[] segmentUpdateDetails) {
-    this.updateDetails = segmentUpdateDetails;
-  }
-
-  /**
-   * This will return the lock object used to lock the table update status file before updation.
-   *
-   * @return
-   */
-  public ICarbonLock getTableUpdateStatusLock() {
-    return CarbonLockFactory.getCarbonLockObj(absoluteTableIdentifier.getCarbonTableIdentifier(),
-        LockUsage.TABLE_UPDATE_STATUS_LOCK);
-  }
-
-  /**
-   * Returns all delete delta files of specified block
-   *
-   * @param tupleId
-   * @return
-   * @throws Exception
-   */
-  public List<String> getDeleteDeltaFiles(String tupleId) throws Exception {
-    return getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT);
-  }
-
-
-  /**
-   * Returns all update delta files of specified Segment.
-   *
-   * @param segmentId
-   * @return
-   * @throws Exception
-   */
-  public List<String> getUpdateDeltaFiles(final String segmentId) {
-    List<String> updatedDeltaFilesList =
-        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-    String endTimeStamp = "";
-    String startTimeStamp = "";
-    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
-    CarbonFile segDir =
-        FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
-    for (LoadMetadataDetails eachSeg : segmentDetails) {
-      if (eachSeg.getLoadName().equalsIgnoreCase(segmentId)) {
-        // if the segment is found then take the start and end time stamp.
-        startTimeStamp = eachSeg.getUpdateDeltaStartTimestamp();
-        endTimeStamp = eachSeg.getUpdateDeltaEndTimestamp();
-      }
-    }
-    // if start timestamp is empty then no update delta is found. so return empty list.
-    if (startTimeStamp.isEmpty()) {
-      return updatedDeltaFilesList;
-    }
-    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimeStamp);
-    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimeStamp);
-
-    // else scan the segment for the delta files with the respective timestamp.
-    CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {
-
-      @Override public boolean accept(CarbonFile pathName) {
-        String fileName = pathName.getName();
-        if (fileName.endsWith(CarbonCommonConstants.UPDATE_DELTA_FILE_EXT)) {
-          String firstPart = fileName.substring(0, fileName.indexOf('.'));
-
-          long timestamp = Long.parseLong(firstPart
-              .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
-                  firstPart.length()));
-          if (Long.compare(timestamp, endTimeStampFinal) <= 0
-              && Long.compare(timestamp, startTimeStampFinal) >= 0) {
-
-            // if marked for delete then it is invalid.
-            if (!isBlockValid(segmentId, fileName)) {
-              return false;
-            }
-
-            return true;
-          }
-        }
-        return false;
-      }
-    });
-
-    for (CarbonFile cfile : files) {
-      updatedDeltaFilesList.add(cfile.getCanonicalPath());
-    }
-
-    return updatedDeltaFilesList;
-  }
-
-  /**
-   * Returns all deleted records of specified block
-   *
-   * @param tupleId
-   * @return
-   * @throws Exception
-   */
-  public int[] getDeleteDeltaDataFromAllFiles(String tupleId) throws Exception {
-    List<String> deltaFiles = getDeltaFiles(tupleId, CarbonCommonConstants.DELETE_DELTA_FILE_EXT);
-    CarbonDeleteFilesDataReader dataReader = new CarbonDeleteFilesDataReader();
-    String blockletId = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCKLET_ID);
-    return dataReader.getDeleteDataFromAllFiles(deltaFiles, blockletId);
-  }
-
-
-
-  /**
-   * Returns all delta file paths of specified block
-   *
-   * @param tupleId
-   * @param extension
-   * @return
-   * @throws Exception
-   */
-  public List<String> getDeltaFiles(String tupleId, String extension) throws Exception {
-    try {
-      CarbonTablePath carbonTablePath = CarbonStorePath
-          .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-              absoluteTableIdentifier.getCarbonTableIdentifier());
-      String segment = CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID);
-      String carbonDataDirectoryPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment);
-      String completeBlockName = CarbonTablePath.addDataPartPrefix(
-          CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.BLOCK_ID)
-              + CarbonCommonConstants.FACT_FILE_EXT);
-      String blockPath =
-          carbonDataDirectoryPath + CarbonCommonConstants.FILE_SEPARATOR + completeBlockName;
-      CarbonFile file = FileFactory.getCarbonFile(blockPath, FileFactory.getFileType(blockPath));
-      if (!file.exists()) {
-        throw new Exception("Invalid tuple id " + tupleId);
-      }
-      String blockNameWithoutExtn = completeBlockName.substring(0, completeBlockName.indexOf('.'));
-      //blockName without timestamp
-      final String blockNameFromTuple =
-          blockNameWithoutExtn.substring(0, blockNameWithoutExtn.lastIndexOf("-"));
-      SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray =
-          readLoadMetadata();
-      return getDeltaFiles(file, blockNameFromTuple, listOfSegmentUpdateDetailsArray, extension,
-          segment);
-
-    } catch (Exception ex) {
-      String errorMsg = "Invalid tuple id " + tupleId;
-      LOG.error(errorMsg);
-      throw new Exception(errorMsg);
-    }
-  }
-
-  /**
-   * This method returns the list of Blocks associated with the segment
-   * from the SegmentUpdateDetails List.
-   * @param segmentName
-   * @return
-   */
-  public List<String> getBlockNameFromSegment(String segmentName) {
-    List<String> blockNames = new ArrayList<String>();
-    for (SegmentUpdateDetails block : updateDetails) {
-      if (block.getSegmentName().equalsIgnoreCase(segmentName) && !CarbonUpdateUtil
-          .isBlockInvalid(block.getStatus())) {
-        blockNames.add(block.getBlockName());
-      }
-    }
-    return blockNames;
-  }
-
-  /**
-   *
-   * @param segName
-   * @param blockName
-   * @return
-   */
-  public boolean isBlockValid(String segName, String blockName) {
-
-    SegmentUpdateDetails details = getDetailsForABlock(segName, blockName);
-
-    if (details == null || !CarbonUpdateUtil.isBlockInvalid(details.getStatus())) {
-      return true;
-    }
-
-    return false;
-  }
-  /**
-   * Returns all delta file paths of specified block
-   *
-   * @param blockDir
-   * @param blockNameFromTuple
-   * @param listOfSegmentUpdateDetailsArray
-   * @param extension
-   * @return
-   */
-  public List<String> getDeltaFiles(CarbonFile blockDir, final String blockNameFromTuple,
-      SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray,
-      final String extension,
-      String segment) {
-    List<String> deleteFileList = null;
-    for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) {
-      if (block.getBlockName().equalsIgnoreCase(blockNameFromTuple) && block.getSegmentName()
-          .equalsIgnoreCase(segment) && !CarbonUpdateUtil.isBlockInvalid(block.getStatus())) {
-        final long deltaStartTimestamp = getStartTimeOfDeltaFile(extension, block);
-        // If there is no delete delete file , then return null
-        if(deltaStartTimestamp == 0) {
-          return deleteFileList;
-        }
-        final long deltaEndTimeStamp = getEndTimeOfDeltaFile(extension, block);
-
-        // final long deltaEndTimeStamp = block.getDeleteDeltaEndTimeAsLong();
-        // final long deltaStartTimestamp = block.getDeleteDeltaStartTimeAsLong();
-        return getFilePaths(blockDir, blockNameFromTuple, extension, deleteFileList,
-            deltaStartTimestamp, deltaEndTimeStamp);
-      }
-    }
-    return deleteFileList;
-  }
-
-  private List<String> getFilePaths(CarbonFile blockDir, final String blockNameFromTuple,
-      final String extension, List<String> deleteFileList, final long deltaStartTimestamp,
-      final long deltaEndTimeStamp) {
-    CarbonFile[] files = blockDir.getParentFile().listFiles(new CarbonFileFilter() {
-
-      @Override public boolean accept(CarbonFile pathName) {
-        String fileName = pathName.getName();
-        if (fileName.endsWith(extension)) {
-          String firstPart = fileName.substring(0, fileName.indexOf('.'));
-          String blockName =
-              firstPart.substring(0, firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN));
-          long timestamp = Long.parseLong(firstPart
-              .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
-                  firstPart.length()));
-          if (blockNameFromTuple.equals(blockName) && ((
-              Long.compare(timestamp, deltaEndTimeStamp ) <= 0) && (
-              Long.compare(timestamp, deltaStartTimestamp) >= 0))) {
-            return true;
-          }
-        }
-        return false;
-      }
-    });
-
-    for (CarbonFile cfile : files) {
-      if(null == deleteFileList) {
-        deleteFileList = new ArrayList<String>(files.length);
-      }
-      deleteFileList.add(cfile.getCanonicalPath());
-    }
-    return deleteFileList;
-  }
-
-  /**
-   * Return all delta file for a block.
-   * @param segmentId
-   * @param blockName
-   * @return
-   */
-  public CarbonFile[] getDeleteDeltaFilesList(final String segmentId, final String blockName) {
-
-    CarbonTablePath carbonTablePath = CarbonStorePath
-        .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-            absoluteTableIdentifier.getCarbonTableIdentifier());
-
-    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
-
-    CarbonFile segDir =
-        FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
-
-    for (SegmentUpdateDetails block : updateDetails) {
-      if ((block.getBlockName().equalsIgnoreCase(blockName)) &&
-          (block.getSegmentName().equalsIgnoreCase(segmentId))
-          && !CarbonUpdateUtil.isBlockInvalid((block.getStatus()))) {
-        final long deltaStartTimestamp =
-            getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
-        final long deltaEndTimeStamp =
-            getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
-        CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {
-
-          @Override public boolean accept(CarbonFile pathName) {
-            String fileName = pathName.getName();
-            if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
-              String firstPart = fileName.substring(0, fileName.indexOf('.'));
-              String blkName = firstPart.substring(0, firstPart.lastIndexOf("-"));
-              long timestamp = Long.parseLong(
-                  firstPart.substring(firstPart.lastIndexOf("-") + 1, firstPart.length()));
-              if (blockName.equals(blkName) && (Long.compare(timestamp, deltaEndTimeStamp) <= 0)
-                  && (Long.compare(timestamp, deltaStartTimestamp) >= 0)) {
-                return true;
-              }
-            }
-            return false;
-          }
-        });
-
-        return files;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Returns all update delta files of specified Segment.
-   *
-   * @param segmentId
-   * @param validUpdateFiles if true then only the valid range files will be returned.
-   * @return
-   */
-  public CarbonFile[] getUpdateDeltaFilesList(String segmentId, final boolean validUpdateFiles,
-      final String fileExtension, final boolean excludeOriginalFact,
-      CarbonFile[] allFilesOfSegment) {
-
-    CarbonTablePath carbonTablePath = CarbonStorePath
-        .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-            absoluteTableIdentifier.getCarbonTableIdentifier());
-    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
-    String endTimeStamp = "";
-    String startTimeStamp = "";
-    long factTimeStamp = 0;
-
-    LoadMetadataDetails[] segmentDetails =
-        segmentStatusManager.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
-
-    for (LoadMetadataDetails eachSeg : segmentDetails) {
-      if (eachSeg.getLoadName().equalsIgnoreCase(segmentId)) {
-        // if the segment is found then take the start and end time stamp.
-        startTimeStamp = eachSeg.getUpdateDeltaStartTimestamp();
-        endTimeStamp = eachSeg.getUpdateDeltaEndTimestamp();
-        factTimeStamp = eachSeg.getLoadStartTime();
-      }
-    }
-
-    // if start timestamp is empty then no update delta is found. so return empty list.
-    if (startTimeStamp.isEmpty()) {
-      return new CarbonFile[0];
-    }
-
-    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimeStamp);
-    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimeStamp);
-    final long factTimeStampFinal = factTimeStamp;
-
-    List<CarbonFile> listOfCarbonFiles =
-        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
-    // else scan the segment for the delta files with the respective timestamp.
-
-    for (CarbonFile eachFile : allFilesOfSegment) {
-
-      String fileName = eachFile.getName();
-      if (fileName.endsWith(fileExtension)) {
-        String firstPart = fileName.substring(0, fileName.indexOf('.'));
-
-        long timestamp = Long.parseLong(firstPart
-            .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
-                firstPart.length()));
-
-        if (excludeOriginalFact) {
-          if (Long.compare(factTimeStampFinal, timestamp) == 0) {
-            continue;
-          }
-        }
-
-        if (validUpdateFiles) {
-          if (Long.compare(timestamp, endTimeStampFinal) <= 0
-              && Long.compare(timestamp, startTimeStampFinal) >= 0) {
-            listOfCarbonFiles.add(eachFile);
-          }
-        } else {
-          // invalid cases.
-          if (Long.compare(timestamp, startTimeStampFinal) < 0) {
-            listOfCarbonFiles.add(eachFile);
-          }
-        }
-      }
-    }
-
-    return listOfCarbonFiles.toArray(new CarbonFile[listOfCarbonFiles.size()]);
-  }
-
-  /**
-   * Returns all update delta files of specified Segment.
-   *
-   * @param segmentId
-   * @param validUpdateFiles
-   * @param fileExtension
-   * @param excludeOriginalFact
-   * @param allFilesOfSegment
-   * @return
-   */
-  public CarbonFile[] getUpdateDeltaFilesForSegment(String segmentId,
-      final boolean validUpdateFiles, final String fileExtension, final boolean excludeOriginalFact,
-      CarbonFile[] allFilesOfSegment) {
-
-    String endTimeStamp = "";
-    String startTimeStamp = "";
-    long factTimeStamp = 0;
-
-    for (LoadMetadataDetails eachSeg : segmentDetails) {
-      if (eachSeg.getLoadName().equalsIgnoreCase(segmentId)) {
-        // if the segment is found then take the start and end time stamp.
-        startTimeStamp = eachSeg.getUpdateDeltaStartTimestamp();
-        endTimeStamp = eachSeg.getUpdateDeltaEndTimestamp();
-        factTimeStamp = eachSeg.getLoadStartTime();
-      }
-    }
-
-    // if start timestamp is empty then no update delta is found. so return empty list.
-    if (startTimeStamp.isEmpty()) {
-      return new CarbonFile[0];
-    }
-
-    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimeStamp);
-    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimeStamp);
-    final long factTimeStampFinal = factTimeStamp;
-
-    List<CarbonFile> listOfCarbonFiles =
-        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
-    // else scan the segment for the delta files with the respective timestamp.
-
-    for (CarbonFile eachFile : allFilesOfSegment) {
-
-      String fileName = eachFile.getName();
-      if (fileName.endsWith(fileExtension)) {
-        String firstPart = fileName.substring(0, fileName.indexOf('.'));
-
-        long timestamp = Long.parseLong(firstPart
-            .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
-                firstPart.length()));
-
-        if (excludeOriginalFact) {
-          if (Long.compare(factTimeStampFinal, timestamp) == 0) {
-            continue;
-          }
-        }
-
-        if (validUpdateFiles) {
-          if (Long.compare(timestamp, endTimeStampFinal) <= 0
-              && Long.compare(timestamp, startTimeStampFinal) >= 0) {
-
-            boolean validBlock = true;
-
-            for(SegmentUpdateDetails blockDetails : getUpdateStatusDetails()) {
-              if (blockDetails.getActualBlockName().equalsIgnoreCase(eachFile.getName())
-                  && CarbonUpdateUtil.isBlockInvalid(blockDetails.getStatus())) {
-                validBlock = false;
-              }
-            }
-
-            if (validBlock) {
-              listOfCarbonFiles.add(eachFile);
-            }
-
-          }
-        } else {
-          // invalid cases.
-          if (Long.compare(timestamp, startTimeStampFinal) < 0) {
-            listOfCarbonFiles.add(eachFile);
-          }
-        }
-      }
-    }
-
-    return listOfCarbonFiles.toArray(new CarbonFile[listOfCarbonFiles.size()]);
-  }
-
-  /**
-   *
-   * @param extension
-   * @param block
-   * @return
-   */
-  private long getStartTimeOfDeltaFile(String extension, SegmentUpdateDetails block) {
-    long startTimestamp;
-    switch (extension) {
-      case CarbonCommonConstants.DELETE_DELTA_FILE_EXT:
-        startTimestamp = block.getDeleteDeltaStartTimeAsLong();
-        break;
-      default:
-        startTimestamp = 0;
-    }
-    return startTimestamp;
-  }
-
-  /**
-   *
-   * @param extension
-   * @param block
-   * @return
-   */
-  private long getEndTimeOfDeltaFile(String extension, SegmentUpdateDetails block) {
-    long endTimestamp;
-    switch (extension) {
-      case CarbonCommonConstants.DELETE_DELTA_FILE_EXT:
-        endTimestamp = block.getDeleteDeltaEndTimeAsLong();
-        break;
-      default: endTimestamp = 0;
-    }
-    return endTimestamp;
-  }
-
-
-  /**
-   * This method loads segment update details
-   *
-   * @return
-   */
-  public SegmentUpdateDetails[] readLoadMetadata() {
-    Gson gsonObjectToRead = new Gson();
-    DataInputStream dataInputStream = null;
-    BufferedReader buffReader = null;
-    InputStreamReader inStream = null;
-    SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray;
-
-    // get the updated status file identifier from the table status.
-    String tableUpdateStatusIdentifier = getUpdatedStatusIdentifier();
-
-    if(null == tableUpdateStatusIdentifier) {
-      return new SegmentUpdateDetails[0];
-    }
-
-    CarbonTablePath carbonTablePath = CarbonStorePath
-        .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-            absoluteTableIdentifier.getCarbonTableIdentifier());
-
-    String tableUpdateStatusPath =
-        carbonTablePath.getMetadataDirectoryPath() + CarbonCommonConstants.FILE_SEPARATOR
-            + tableUpdateStatusIdentifier;
-    AtomicFileOperations fileOperation = new AtomicFileOperationsImpl(tableUpdateStatusPath,
-        FileFactory.getFileType(tableUpdateStatusPath));
-
-    try {
-      if (!FileFactory
-          .isFileExist(tableUpdateStatusPath, FileFactory.getFileType(tableUpdateStatusPath))) {
-        return new SegmentUpdateDetails[0];
-      }
-      dataInputStream = fileOperation.openForRead();
-      inStream = new InputStreamReader(dataInputStream,
-          CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
-      buffReader = new BufferedReader(inStream);
-      listOfSegmentUpdateDetailsArray =
-          gsonObjectToRead.fromJson(buffReader, SegmentUpdateDetails[].class);
-    } catch (IOException e) {
-      return new SegmentUpdateDetails[0];
-    } finally {
-      closeStreams(buffReader, inStream, dataInputStream);
-    }
-
-    return listOfSegmentUpdateDetailsArray;
-  }
-
-  /**
-   * @return updateStatusFileName
-   */
-  private String getUpdatedStatusIdentifier() {
-    SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
-    CarbonTablePath carbonTablePath = CarbonStorePath
-        .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-            absoluteTableIdentifier.getCarbonTableIdentifier());
-    LoadMetadataDetails[] loadDetails =
-        ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
-    if (loadDetails.length == 0) {
-      return null;
-    }
-    return loadDetails[0].getUpdateStatusFileName();
-  }
-
-  /**
-   * writes segment update details into a given file at @param dataLoadLocation
-   *
-   * @param listOfSegmentUpdateDetailsArray
-   * @throws IOException
-   */
-  public void writeLoadDetailsIntoFile(List<SegmentUpdateDetails> listOfSegmentUpdateDetailsArray,
-      String updateStatusFileIdentifier) throws IOException {
-
-    CarbonTablePath carbonTablePath = CarbonStorePath
-        .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-            absoluteTableIdentifier.getCarbonTableIdentifier());
-
-    String fileLocation =
-        carbonTablePath.getMetadataDirectoryPath() + CarbonCommonConstants.FILE_SEPARATOR
-            + CarbonUpdateUtil.getUpdateStatusFileName(updateStatusFileIdentifier);
-
-    AtomicFileOperations fileWrite =
-        new AtomicFileOperationsImpl(fileLocation, FileFactory.getFileType(fileLocation));
-    BufferedWriter brWriter = null;
-    DataOutputStream dataOutputStream = null;
-    Gson gsonObjectToWrite = new Gson();
-    // write the updated data into the metadata file.
-
-    try {
-      dataOutputStream = fileWrite.openForWrite(FileWriteOperation.OVERWRITE);
-      brWriter = new BufferedWriter(new OutputStreamWriter(dataOutputStream,
-          CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT));
-
-      String metadataInstance = gsonObjectToWrite.toJson(listOfSegmentUpdateDetailsArray);
-      brWriter.write(metadataInstance);
-    } catch (IOException ioe) {
-      LOG.error("Error message: " + ioe.getLocalizedMessage());
-    } finally {
-      if (null != brWriter) {
-        brWriter.flush();
-      }
-      CarbonUtil.closeStreams(brWriter);
-      fileWrite.close();
-    }
-
-  }
-
-  /**
-   * compares passed time stamp with status file delete timestamp and
-   * returns latest timestamp from status file if both are not equal
-   * returns null otherwise
-   *
-   * @param completeBlockName
-   * @param timestamp
-   * @return
-   */
-  public String getTimestampForRefreshCache(String completeBlockName, String timestamp) {
-    long cacheTimestamp = 0;
-    if (null != timestamp) {
-      cacheTimestamp = CarbonUpdateUtil.getTimeStampAsLong(timestamp);
-    }
-    String blockName = CarbonTablePath.addDataPartPrefix(CarbonUpdateUtil.getBlockName(
-        CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.BLOCK_ID)));
-    String segmentId =
-        CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.SEGMENT_ID);
-    CarbonTablePath carbonTablePath = CarbonStorePath
-        .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-            absoluteTableIdentifier.getCarbonTableIdentifier());
-    SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray =
-        readLoadMetadata();
-    for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) {
-      if (segmentId.equalsIgnoreCase(block.getSegmentName()) && block.getBlockName()
-          .equalsIgnoreCase(blockName) && !CarbonUpdateUtil.isBlockInvalid(block.getStatus())) {
-        long deleteTimestampFromStatusFile = block.getDeleteDeltaEndTimeAsLong();
-        if (Long.compare(deleteTimestampFromStatusFile, cacheTimestamp) == 0) {
-          return null;
-        } else {
-          return block.getDeleteDeltaEndTimestamp();
-        }
-      }
-    }
-    return null;
-  }
-
-  /**
-   * This method closes the streams
-   *
-   * @param streams - streams to close.
-   */
-  private void closeStreams(Closeable... streams) {
-    // Added if to avoid NullPointerException in case one stream is being passed as null
-    if (null != streams) {
-      for (Closeable stream : streams) {
-        if (null != stream) {
-          try {
-            stream.close();
-          } catch (IOException e) {
-            LOG.error("Error while closing stream" + stream);
-          }
-        }
-      }
-    }
-  }
-  /**
-   * Get the invalid tasks in that segment.
-   * @param segmentId
-   * @return
-   */
-  public List<String> getInvalidBlockList(String segmentId) {
-
-    // get the original fact file timestamp from the table status file.
-    List<String> listOfInvalidBlocks = new ArrayList<String>();
-    SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);
-    CarbonTablePath carbonTablePath = CarbonStorePath
-        .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-            absoluteTableIdentifier.getCarbonTableIdentifier());
-    LoadMetadataDetails[] segmentDetails =
-        ssm.readLoadMetadata(carbonTablePath.getMetadataDirectoryPath());
-    long timestampOfOriginalFacts = 0;
-
-    String startTimestampOfUpdate = "" ;
-    String endTimestampOfUpdate = "";
-
-    for(LoadMetadataDetails segment : segmentDetails){
-      // find matching segment and return timestamp.
-      if (segment.getLoadName().equalsIgnoreCase(segmentId)) {
-        timestampOfOriginalFacts = segment.getLoadStartTime();
-        startTimestampOfUpdate = segment.getUpdateDeltaStartTimestamp();
-        endTimestampOfUpdate = segment.getUpdateDeltaEndTimestamp();
-      }
-    }
-
-    if (startTimestampOfUpdate.isEmpty()) {
-      return listOfInvalidBlocks;
-
-    }
-
-    // now after getting the original fact timestamp, what ever is remaining
-    // files need to cross check it with table status file.
-
-    // filter out the fact files.
-
-    String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segmentId);
-    CarbonFile segDir =
-        FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath));
-
-    final Long endTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(endTimestampOfUpdate);
-    final Long startTimeStampFinal = CarbonUpdateUtil.getTimeStampAsLong(startTimestampOfUpdate);
-    final Long timeStampOriginalFactFinal =
-        timestampOfOriginalFacts;
-
-    CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {
-
-      @Override public boolean accept(CarbonFile pathName) {
-        String fileName = pathName.getName();
-        if (fileName.endsWith(CarbonCommonConstants.UPDATE_DELTA_FILE_EXT)) {
-          String firstPart = fileName.substring(0, fileName.indexOf('.'));
-
-          long timestamp = Long.parseLong(firstPart
-              .substring(firstPart.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
-                  firstPart.length()));
-          if (Long.compare(timestamp, endTimeStampFinal) <= 0
-              && Long.compare(timestamp, startTimeStampFinal) >= 0) {
-            return false;
-          }
-          if (Long.compare(timestamp, timeStampOriginalFactFinal) == 0) {
-            return false;
-          }
-          // take the rest of files as they are invalid.
-          return true;
-        }
-        return false;
-      }
-    });
-
-    // gather the task numbers.
-    for(CarbonFile updateFiles : files) {
-      listOfInvalidBlocks.add(updateFiles.getName());
-    }
-
-    return listOfInvalidBlocks;
-  }
-  /**
-   * Returns the invalid timestamp range of a segment.
-   * @param segmentId
-   * @return
-   */
-  public UpdateVO getInvalidTimestampRange(String segmentId) {
-    UpdateVO range = new UpdateVO();
-    for (LoadMetadataDetails segment : segmentDetails) {
-      if (segment.getLoadName().equalsIgnoreCase(segmentId)) {
-        range.setSegmentId(segmentId);
-        range.setFactTimestamp(segment.getLoadStartTime());
-        if (!segment.getUpdateDeltaStartTimestamp().isEmpty() && !segment
-            .getUpdateDeltaEndTimestamp().isEmpty()) {
-          range.setUpdateDeltaStartTimestamp(
-              CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaStartTimestamp()));
-          range.setLatestUpdateTimestamp(
-              CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaEndTimestamp()));
-        }
-      }
-    }
-    return range;
-  }
-  /**
-   *
-   * @param segmentId
-   * @param block
-   * @param needCompleteList
-   * @return
-   */
-  public CarbonFile[] getDeleteDeltaInvalidFilesList(final String segmentId,
-      final SegmentUpdateDetails block, final boolean needCompleteList,
-      CarbonFile[] allSegmentFiles) {
-
-    final long deltaStartTimestamp =
-        getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
-
-    List<CarbonFile> files =
-        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
-    for (CarbonFile eachFile : allSegmentFiles) {
-      String fileName = eachFile.getName();
-      if (fileName.endsWith(CarbonCommonConstants.DELETE_DELTA_FILE_EXT)) {
-        String blkName = CarbonTablePath.DataFileUtil.getBlockNameFromDeleteDeltaFile(fileName);
-
-        // complete list of delta files of that block is returned.
-        if (needCompleteList && block.getBlockName().equalsIgnoreCase(blkName)) {
-          files.add(eachFile);
-        }
-
-        // invalid delete delta files only will be returned.
-        long timestamp = CarbonUpdateUtil.getTimeStampAsLong(
-            CarbonTablePath.DataFileUtil.getTimeStampFromDeleteDeltaFile(fileName));
-
-        if (block.getBlockName().equalsIgnoreCase(blkName) && (
-            Long.compare(timestamp, deltaStartTimestamp) < 0)) {
-          files.add(eachFile);
-        }
-      }
-    }
-
-    return files.toArray(new CarbonFile[files.size()]);
-  }
-
-  /**
-   *
-   * @param blockName
-   * @param allSegmentFiles
-   * @return
-   */
-  public CarbonFile[] getAllBlockRelatedFiles(String blockName, CarbonFile[] allSegmentFiles,
-                                              String actualBlockName) {
-    List<CarbonFile> files = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
-    for (CarbonFile eachFile : allSegmentFiles) {
-
-      // for carbon data.
-      if (eachFile.getName().equalsIgnoreCase(actualBlockName)) {
-        files.add(eachFile);
-      }
-
-      // get carbon index files of the block.
-      String taskNum = CarbonTablePath.DataFileUtil.getTaskNo(actualBlockName);
-      // String indexFileEndsWith = timestamp + CarbonTablePath.getCarbonIndexExtension();
-      if (eachFile.getName().endsWith(CarbonTablePath.getCarbonIndexExtension()) && eachFile
-          .getName().startsWith(taskNum)) {
-        files.add(eachFile);
-      }
-
-    }
-
-    return files.toArray(new CarbonFile[files.size()]);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 11fa4c1..189ca4d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -24,21 +24,21 @@ import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.List;
 
-import org.apache.carbondata.core.carbon.ColumnarFormatVersion;
-import org.apache.carbondata.core.carbon.datastore.block.BlockInfo;
-import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.carbon.metadata.blocklet.SegmentInfo;
-import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
-import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.PresenceMeta;
-import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletBTreeIndex;
-import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex;
-import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletMinMaxIndex;
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.BlockInfo;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
+import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.reader.CarbonIndexFileReader;
 import org.apache.carbondata.format.BlockIndex;
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 98eaedf..c85e12d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -30,14 +30,14 @@ import java.util.Set;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.carbon.ColumnarFormatVersion;
-import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.carbon.metadata.index.BlockIndexInfo;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
 import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
 import org.apache.carbondata.format.BlockIndex;
 import org.apache.carbondata.format.BlockletBTreeIndex;
 import org.apache.carbondata.format.BlockletIndex;
@@ -134,7 +134,7 @@ public class CarbonMetadataUtil {
   }
 
   private static BlockletIndex getBlockletIndex(
-      org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex info) {
+      org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex info) {
     BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
 
     for (int i = 0; i < info.getMinMaxIndex().getMaxValues().length; i++) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 0763f1b..5a32a22 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -27,8 +27,8 @@ import java.util.Properties;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.carbon.ColumnarFormatVersion;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 
 public final class CarbonProperties {
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
index ce166fd..90b2317 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
@@ -17,12 +17,12 @@
 
 package org.apache.carbondata.core.util;
 
-import org.apache.carbondata.core.carbon.querystatistics.DriverQueryStatisticsRecorderDummy;
-import org.apache.carbondata.core.carbon.querystatistics.DriverQueryStatisticsRecorderImpl;
-import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorder;
-import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorderDummy;
-import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorderImpl;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.stats.DriverQueryStatisticsRecorderDummy;
+import org.apache.carbondata.core.stats.DriverQueryStatisticsRecorderImpl;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorderDummy;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorderImpl;
 
 public class CarbonTimeStatisticsFactory {
   private static String loadStatisticsInstanceType;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index e61d1d2..aca8ae9 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -40,42 +40,42 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.cache.dictionary.Dictionary;
 import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.carbon.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.carbon.path.CarbonStorePath;
-import org.apache.carbondata.core.carbon.path.CarbonTablePath;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
-import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
-import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.mutate.UpdateVO;
 import org.apache.carbondata.core.reader.ThriftReader;
 import org.apache.carbondata.core.reader.ThriftReader.TBaseCreator;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
 import org.apache.carbondata.core.service.PathService;
-import org.apache.carbondata.core.update.UpdateVO;
-import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
+import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
+import org.apache.carbondata.core.util.path.CarbonStorePath;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
 import org.apache.carbondata.format.DataChunk2;
-import org.apache.carbondata.scan.model.QueryDimension;
 
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -1064,7 +1064,7 @@ public final class CarbonUtil {
     int counter = 0;
     for (int i = 0; i < wrapperColumnSchemaList.size(); i++) {
       if (CarbonUtil.hasEncoding(wrapperColumnSchemaList.get(i).getEncodingList(),
-          org.apache.carbondata.core.carbon.metadata.encoder.Encoding.DICTIONARY)) {
+          org.apache.carbondata.core.metadata.encoder.Encoding.DICTIONARY)) {
         cardinality.add(dictionaryColumnCardinality[counter]);
         counter++;
       } else if (!wrapperColumnSchemaList.get(i).isDimensionColumn()) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
index e766e85..2615e6d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
@@ -23,15 +23,15 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import org.apache.carbondata.core.carbon.ColumnarFormatVersion;
-import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.carbon.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
-import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.reader.CarbonFooterReader;
 import org.apache.carbondata.format.FileFooter;
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
index 02de383..0d71d7e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
@@ -22,12 +22,12 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.carbondata.core.carbon.ColumnarFormatVersion;
-import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.carbon.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter;
-import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.reader.CarbonFooterReader;
 import org.apache.carbondata.format.FileFooter;
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
index dc79710..92a507a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
@@ -18,7 +18,7 @@
  */
 package org.apache.carbondata.core.util;
 
-import org.apache.carbondata.core.carbon.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 
 /**
  * Factory class to get the thrift reader object based on version

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index c9ed234..88eb5f1 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -31,10 +31,10 @@ import java.util.Map;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
 
 import org.apache.spark.unsafe.types.UTF8String;
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index a9be95c..1de0139 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -27,14 +27,14 @@ import org.apache.carbondata.core.compression.BigDecimalCompressor;
 import org.apache.carbondata.core.compression.BigIntCompressor;
 import org.apache.carbondata.core.compression.DoubleCompressor;
 import org.apache.carbondata.core.compression.ValueCompressor;
-import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.store.compression.ReaderCompressModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.compression.decimal.*;
-import org.apache.carbondata.core.datastorage.store.compression.nondecimal.*;
-import org.apache.carbondata.core.datastorage.store.compression.none.*;
-import org.apache.carbondata.core.datastorage.store.compression.type.*;
+import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastore.compression.ReaderCompressModel;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.compression.decimal.*;
+import org.apache.carbondata.core.datastore.compression.nondecimal.*;
+import org.apache.carbondata.core.datastore.compression.none.*;
+import org.apache.carbondata.core.datastore.compression.type.*;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
 
 public final class ValueCompressionUtil {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/path/CarbonSharedDictionaryPath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonSharedDictionaryPath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonSharedDictionaryPath.java
new file mode 100644
index 0000000..c27b6bd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonSharedDictionaryPath.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.util.path;
+
+import java.io.File;
+
+/**
+ * Helps to get Shared dimension files path.
+ */
+public class CarbonSharedDictionaryPath {
+
+  private static final String SHAREDDIM_DIR = "SharedDictionary";
+  private static final String DICTIONARY_EXT = ".dict";
+  private static final String DICTIONARY_META_EXT = ".dictmeta";
+  private static final String SORT_INDEX_EXT = ".sortindex";
+
+  /***
+   * @param storePath    store path
+   * @param databaseName data base name
+   * @param columnId     unique column identifier
+   * @return absolute path of shared dictionary file
+   */
+  public static String getDictionaryFilePath(String storePath, String databaseName,
+      String columnId) {
+    return getSharedDictionaryDir(storePath, databaseName) + File.separator + columnId
+        + DICTIONARY_EXT;
+  }
+
+  /***
+   * @param storePath    store path
+   * @param databaseName data base name
+   * @param columnId     unique column identifier
+   * @return absolute path of shared dictionary meta file
+   */
+  public static String getDictionaryMetaFilePath(String storePath, String databaseName,
+      String columnId) {
+    return getSharedDictionaryDir(storePath, databaseName) + File.separator + columnId
+        + DICTIONARY_META_EXT;
+  }
+
+  /***
+   * @param storePath    store path
+   * @param databaseName data base name
+   * @param columnId     unique column identifier
+   * @return absolute path of shared dictionary sort index file
+   */
+  public static String getSortIndexFilePath(String storePath, String databaseName,
+      String columnId) {
+    return getSharedDictionaryDir(storePath, databaseName) + File.separator + columnId
+        + SORT_INDEX_EXT;
+  }
+
+  private static String getSharedDictionaryDir(String storePath, String databaseName) {
+    return storePath + File.separator + databaseName + File.separator + SHAREDDIM_DIR;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
new file mode 100644
index 0000000..7d435a1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.util.path;
+
+import java.io.File;
+
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Helps to get Store content paths.
+ */
+public class CarbonStorePath extends Path {
+
+  private String storePath;
+
+  public CarbonStorePath(String storePathString) {
+    super(storePathString);
+    this.storePath = storePathString;
+  }
+
+  /**
+   * gets CarbonTablePath object to manage table paths
+   */
+  public static CarbonTablePath getCarbonTablePath(String storePath,
+      CarbonTableIdentifier tableIdentifier) {
+    CarbonTablePath carbonTablePath = new CarbonTablePath(tableIdentifier,
+        storePath + File.separator + tableIdentifier.getDatabaseName() + File.separator
+            + tableIdentifier.getTableName());
+    return carbonTablePath;
+  }
+
+  public static CarbonTablePath getCarbonTablePath(String storePath,
+      String dbName, String tableName) {
+    return new CarbonTablePath(storePath, dbName, tableName);
+  }
+
+  public static CarbonTablePath getCarbonTablePath(AbsoluteTableIdentifier identifier) {
+    CarbonTableIdentifier id = identifier.getCarbonTableIdentifier();
+    return new CarbonTablePath(id, identifier.getTablePath());
+  }
+
+  /**
+   * gets CarbonTablePath object to manage table paths
+   */
+  public CarbonTablePath getCarbonTablePath(CarbonTableIdentifier tableIdentifier) {
+    return CarbonStorePath.getCarbonTablePath(storePath, tableIdentifier);
+  }
+
+  @Override public boolean equals(Object o) {
+    if (!(o instanceof CarbonStorePath)) {
+      return false;
+    }
+    CarbonStorePath path = (CarbonStorePath)o;
+    return storePath.equals(path.storePath) && super.equals(o);
+  }
+
+  @Override public int hashCode() {
+    return super.hashCode() + storePath.hashCode();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
new file mode 100644
index 0000000..7c7f8e4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -0,0 +1,663 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.util.path;
+
+import java.io.File;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+
+import org.apache.hadoop.fs.Path;
+
+
+/**
+ * Helps to get Table content paths.
+ */
+public class CarbonTablePath extends Path {
+
+  protected static final String METADATA_DIR = "Metadata";
+  protected static final String DICTIONARY_EXT = ".dict";
+  protected static final String DICTIONARY_META_EXT = ".dictmeta";
+  protected static final String SORT_INDEX_EXT = ".sortindex";
+  protected static final String SCHEMA_FILE = "schema";
+  protected static final String TABLE_STATUS_FILE = "tablestatus";
+  protected static final String TABLE_UPDATE_STATUS_FILE = "tableupdatestatus";
+  protected static final String FACT_DIR = "Fact";
+  protected static final String SEGMENT_PREFIX = "Segment_";
+  protected static final String PARTITION_PREFIX = "Part";
+  protected static final String CARBON_DATA_EXT = ".carbondata";
+  protected static final String CARBON_DELTE_DELTA_EXT = ".deletedelta";
+  protected static final String CARBON_UPDATE_DELTA_EXT = ".updatedelta";
+  protected static final String DATA_PART_PREFIX = "part-";
+  protected static final String INDEX_FILE_EXT = ".carbonindex";
+  protected static final String DELETE_DELTA_FILE_EXT = ".deletedelta";
+
+  protected String tablePath;
+  protected CarbonTableIdentifier carbonTableIdentifier;
+
+  /**
+   *
+   * @param carbonTableIdentifier
+   * @param tablePathString
+   */
+  public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String tablePathString) {
+    super(tablePathString);
+    this.carbonTableIdentifier = carbonTableIdentifier;
+    this.tablePath = tablePathString;
+  }
+
+  public CarbonTablePath(String storePath, String dbName, String tableName) {
+    super(storePath + File.separator + dbName + File.separator + tableName);
+    this.carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "");
+    this.tablePath = storePath + File.separator + dbName + File.separator + tableName;
+  }
+
+  /**
+   * The method returns the folder path containing the carbon file.
+   *
+   * @param carbonFilePath
+   */
+  public static String getFolderContainingFile(String carbonFilePath) {
+    return carbonFilePath.substring(0, carbonFilePath.lastIndexOf(File.separator));
+  }
+
+  /**
+   * @param columnId unique column identifier
+   * @return name of dictionary file
+   */
+  public static String getDictionaryFileName(String columnId) {
+    return columnId + DICTIONARY_EXT;
+  }
+
+  /**
+   * whether carbonFile is dictionary file or not
+   *
+   * @param carbonFile
+   * @return
+   */
+  public static Boolean isDictionaryFile(CarbonFile carbonFile) {
+    return (!carbonFile.isDirectory()) && (carbonFile.getName().endsWith(DICTIONARY_EXT));
+  }
+
+  /**
+   * check if it is carbon data file matching extension
+   *
+   * @param fileNameWithPath
+   * @return boolean
+   */
+  public static boolean isCarbonDataFile(String fileNameWithPath) {
+    int pos = fileNameWithPath.lastIndexOf('.');
+    if (pos != -1) {
+      return fileNameWithPath.substring(pos).startsWith(CARBON_DATA_EXT);
+    }
+    return false;
+  }
+  /**
+   * check if it is carbon data file matching extension
+   *
+   * @param fileNameWithPath
+   * @return boolean
+   */
+  public static boolean isCarbonDataFileOrUpdateFile(String fileNameWithPath) {
+    int pos = fileNameWithPath.lastIndexOf('.');
+    if (pos != -1) {
+      return fileNameWithPath.substring(pos).startsWith(CARBON_DATA_EXT) || fileNameWithPath
+          .substring(pos).startsWith(CARBON_UPDATE_DELTA_EXT);
+    }
+    return false;
+  }
+
+  /**
+   * check if it is carbon index file matching extension
+   *
+   * @param fileNameWithPath
+   * @return boolean
+   */
+  public static boolean isCarbonIndexFile(String fileNameWithPath) {
+    int pos = fileNameWithPath.lastIndexOf('.');
+    if (pos != -1) {
+      return fileNameWithPath.substring(pos).startsWith(INDEX_FILE_EXT);
+    }
+    return false;
+  }
+
+  /**
+   * gets table path
+   */
+  public String getPath() {
+    return tablePath;
+  }
+
+  /**
+   * @param columnId unique column identifier
+   * @return absolute path of dictionary file
+   */
+  public String getDictionaryFilePath(String columnId) {
+    return getMetaDataDir() + File.separator + getDictionaryFileName(columnId);
+  }
+
+  /**
+   * @return it return relative directory
+   */
+  public String getRelativeDictionaryDirectory() {
+    return carbonTableIdentifier.getDatabaseName() + File.separator + carbonTableIdentifier
+        .getTableName();
+  }
+
+  /**
+   * This method will return the metadata directory location for a table
+   *
+   * @return
+   */
+  public String getMetadataDirectoryPath() {
+    return getMetaDataDir();
+  }
+
+  /**
+   * @param columnId unique column identifier
+   * @return absolute path of dictionary meta file
+   */
+  public String getDictionaryMetaFilePath(String columnId) {
+    return getMetaDataDir() + File.separator + columnId + DICTIONARY_META_EXT;
+  }
+
+  /**
+   * @param columnId unique column identifier
+   * @return absolute path of sort index file
+   */
+  public String getSortIndexFilePath(String columnId) {
+    return getMetaDataDir() + File.separator + columnId + SORT_INDEX_EXT;
+  }
+
+  /**
+   *
+   * @param columnId
+   * @param dictOffset
+   * @return absolute path of sortindex with appeneded dictionary offset
+   */
+  public String getSortIndexFilePath(String columnId, long dictOffset) {
+    return getMetaDataDir() + File.separator + columnId + "_" + dictOffset + SORT_INDEX_EXT;
+  }
+
+  /**
+   * @return absolute path of schema file
+   */
+  public String getSchemaFilePath() {
+    return getMetaDataDir() + File.separator + SCHEMA_FILE;
+  }
+
+  /**
+   * return the schema file path
+   * @param tablePath path to table files
+   * @return schema file path
+   */
+  public static String getSchemaFilePath(String tablePath) {
+    return tablePath + File.separator + METADATA_DIR + File.separator + SCHEMA_FILE;
+  }
+
+  /**
+   * @return absolute path of table status file
+   */
+  public String getTableStatusFilePath() {
+    return getMetaDataDir() + File.separator + TABLE_STATUS_FILE;
+  }
+
+  /**
+   * @return absolute path of table update status file
+   */
+  public String getTableUpdateStatusFilePath() {
+    return getMetaDataDir() + File.separator + TABLE_UPDATE_STATUS_FILE;
+  }
+
+  /**
+   * Gets absolute path of data file
+   *
+   * @param partitionId         unique partition identifier
+   * @param segmentId           unique partition identifier
+   * @param filePartNo          data file part number
+   * @param factUpdateTimeStamp unique identifier to identify an update
+   * @return absolute path of data file stored in carbon data format
+   */
+  public String getCarbonDataFilePath(String partitionId, String segmentId, Integer filePartNo,
+      Integer taskNo, int bucketNumber, String factUpdateTimeStamp) {
+    return getSegmentDir(partitionId, segmentId) + File.separator + getCarbonDataFileName(
+        filePartNo, taskNo, bucketNumber, factUpdateTimeStamp);
+  }
+
+  /**
+   * Below method will be used to get the index file present in the segment folder
+   * based on task id
+   *
+   * @param taskId      task id of the file
+   * @param partitionId partition number
+   * @param segmentId   segment number
+   * @return full qualified carbon index path
+   */
+  public String getCarbonIndexFilePath(final String taskId, final String partitionId,
+      final String segmentId, final String bucketNumber) {
+    String segmentDir = getSegmentDir(partitionId, segmentId);
+    CarbonFile carbonFile =
+        FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
+
+    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
+      @Override public boolean accept(CarbonFile file) {
+        return file.getName().startsWith(taskId + "-" + bucketNumber) && file.getName()
+            .endsWith(INDEX_FILE_EXT);
+      }
+    });
+    if (files.length > 0) {
+      return files[0].getAbsolutePath();
+    } else {
+      throw new RuntimeException("Missing Carbon index file for partition["
+          + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId
+          + "]");
+    }
+  }
+  /**
+   * Below method will be used to get the index file present in the segment folder
+   * based on task id
+   *
+   * @param taskId      task id of the file
+   * @param partitionId partition number
+   * @param segmentId   segment number
+   * @return full qualified carbon index path
+   */
+  public String getCarbonUpdatedIndexFilePath(final String taskId, final String partitionId,
+      final String segmentId) {
+    String segmentDir = getSegmentDir(partitionId, segmentId);
+    CarbonFile carbonFile =
+        FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
+
+    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
+      @Override public boolean accept(CarbonFile file) {
+        return file.getName().startsWith(taskId) && file.getName().endsWith(INDEX_FILE_EXT);
+      }
+    });
+    if (files.length > 0) {
+      return files[0].getAbsolutePath();
+    } else {
+      throw new RuntimeException(
+          "Missing Carbon Updated index file for partition[" + partitionId
+              + "] Segment[" + segmentId + "], taskId[" + taskId + "]");
+    }
+  }
+
+  /**
+   * Below method will be used to get the index file present in the segment folder
+   * based on task id
+   *
+   * @param taskId      task id of the file
+   * @param partitionId partition number
+   * @param segmentId   segment number
+   * @return full qualified carbon index path
+   */
+  public String getCarbonDeleteDeltaFilePath(final String taskId, final String partitionId,
+      final String segmentId) {
+    String segmentDir = getSegmentDir(partitionId, segmentId);
+    CarbonFile carbonFile =
+        FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
+
+    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
+      @Override public boolean accept(CarbonFile file) {
+        return file.getName().startsWith(taskId) && file.getName().endsWith(DELETE_DELTA_FILE_EXT);
+      }
+    });
+    if (files.length > 0) {
+      return files[0].getAbsolutePath();
+    } else {
+      throw new RuntimeException(
+          "Missing Carbon delete delta file index file for partition["
+              + partitionId + "] Segment[" + segmentId + "], taskId[" + taskId
+              + "]");
+    }
+  }
+
+  /**
+   * Gets absolute path of data file
+   *
+   * @param partitionId unique partition identifier
+   * @param segmentId   unique partition identifier
+   * @return absolute path of data file stored in carbon data format
+   */
+  public String getCarbonDataDirectoryPath(String partitionId, String segmentId) {
+    return getSegmentDir(partitionId, segmentId);
+  }
+
+  /**
+   * Gets data file name only with out path
+   *
+   * @param filePartNo          data file part number
+   * @param taskNo              task identifier
+   * @param factUpdateTimeStamp unique identifier to identify an update
+   * @return gets data file name only with out path
+   */
+  public String getCarbonDataFileName(Integer filePartNo, Integer taskNo, int bucketNumber,
+      String factUpdateTimeStamp) {
+    return DATA_PART_PREFIX + filePartNo + "-" + taskNo + "-" + bucketNumber + "-"
+        + factUpdateTimeStamp + CARBON_DATA_EXT;
+  }
+
+  /**
+   * Below method will be used to get the carbon index filename
+   *
+   * @param taskNo               task number
+   * @param factUpdatedTimeStamp time stamp
+   * @return filename
+   */
+  public String getCarbonIndexFileName(int taskNo, int bucketNumber, String factUpdatedTimeStamp) {
+    return taskNo + "-" + bucketNumber + "-" + factUpdatedTimeStamp + INDEX_FILE_EXT;
+  }
+
+  /**
+   * Below method will be used to get the carbon index filename
+   *
+   * @param taskNo               task number
+   * @param factUpdatedTimeStamp time stamp
+   * @return filename
+   */
+  public String getCarbonIndexFileName(int taskNo, String factUpdatedTimeStamp,
+      String indexFileExtension) {
+    return taskNo + "-" + factUpdatedTimeStamp + indexFileExtension;
+  }
+
+  private String getSegmentDir(String partitionId, String segmentId) {
+    return getPartitionDir(partitionId) + File.separator + SEGMENT_PREFIX + segmentId;
+  }
+
+  public String getPartitionDir(String partitionId) {
+    return getFactDir() + File.separator + PARTITION_PREFIX + partitionId;
+  }
+
+  private String getMetaDataDir() {
+    return tablePath + File.separator + METADATA_DIR;
+  }
+
+  public String getFactDir() {
+    return tablePath + File.separator + FACT_DIR;
+  }
+
+  @Override public boolean equals(Object o) {
+    if (!(o instanceof CarbonTablePath)) {
+      return false;
+    }
+    CarbonTablePath path = (CarbonTablePath) o;
+    return tablePath.equals(path.tablePath) && super.equals(o);
+  }
+
+  @Override public int hashCode() {
+    return super.hashCode() + tablePath.hashCode();
+  }
+
+  /**
+   * To manage data file name and composition
+   */
+  public static class DataFileUtil {
+
+    /**
+     * gets updated timestamp information from given carbon data file name
+     */
+    public static String getTimeStampFromFileName(String carbonDataFileName) {
+      // Get the timestamp portion of the file.
+      String fileName = getFileName(carbonDataFileName);
+      int startIndex = fileName.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1;
+      int endIndex = fileName.indexOf(".", startIndex);
+      return fileName.substring(startIndex, endIndex);
+    }
+
+
+    /**
+     * This will return the timestamp present in the delete delta file.
+     * @param fileName
+     * @return
+     */
+    public static String getTimeStampFromDeleteDeltaFile(String fileName) {
+      return fileName.substring(fileName.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
+          fileName.lastIndexOf("."));
+    }
+
+    /**
+     * This will return the timestamp present in the delete delta file.
+     * @param fileName
+     * @return
+     */
+    public static String getBlockNameFromDeleteDeltaFile(String fileName) {
+      return fileName.substring(0,
+          fileName.lastIndexOf(CarbonCommonConstants.HYPHEN));
+    }
+
+    /**
+     * gets updated timestamp information from given carbon data file name
+     */
+    public static String getBucketNo(String carbonFilePath) {
+      // Get the file name from path
+      String fileName = getFileName(carbonFilePath);
+      // + 1 for size of "-"
+      int firstDashPos = fileName.indexOf("-");
+      int secondDash = fileName.indexOf("-", firstDashPos + 1);
+      int startIndex = fileName.indexOf("-", secondDash + 1) + 1;
+      int endIndex = fileName.indexOf("-", startIndex);
+      // to support backward compatibility
+      if (startIndex == -1 || endIndex == -1) {
+        return "0";
+      }
+      return fileName.substring(startIndex, endIndex);
+    }
+
+    /**
+     * gets file part number information from given carbon data file name
+     */
+    public static String getPartNo(String carbonDataFileName) {
+      // Get the file name from path
+      String fileName = getFileName(carbonDataFileName);
+      // + 1 for size of "-"
+      int startIndex = fileName.indexOf("-") + 1;
+      int endIndex = fileName.indexOf("-", startIndex);
+      return fileName.substring(startIndex, endIndex);
+    }
+
+    /**
+     * gets updated timestamp information from given carbon data file name
+     */
+    public static String getTaskNo(String carbonDataFileName) {
+      // Get the file name from path
+      String fileName = getFileName(carbonDataFileName);
+      // + 1 for size of "-"
+      int firstDashPos = fileName.indexOf("-");
+      int startIndex = fileName.indexOf("-", firstDashPos + 1) + 1;
+      int endIndex = fileName.indexOf("-", startIndex);
+      return fileName.substring(startIndex, endIndex);
+    }
+
+    /**
+     * Gets the file name from file path
+     */
+    private static String getFileName(String carbonDataFileName) {
+      int endIndex = carbonDataFileName.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
+      if (endIndex > -1) {
+        return carbonDataFileName.substring(endIndex + 1, carbonDataFileName.length());
+      } else {
+        return carbonDataFileName;
+      }
+    }
+
+    /**
+     * Gets the file name of the delta files.
+     *
+     * @param filePartNo
+     * @param taskNo
+     * @param factUpdateTimeStamp
+     * @param Extension
+     * @return
+     */
+    public static String getCarbonDeltaFileName(String filePartNo, String taskNo,
+        String factUpdateTimeStamp, String Extension) {
+      return DATA_PART_PREFIX + filePartNo + "-" + taskNo + "-" + factUpdateTimeStamp
+          + Extension;
+    }
+  }
+
+  /**
+   * To manage data path and composition
+   */
+  public static class DataPathUtil {
+
+    /**
+     * gets segement id from given absolute data file path
+     */
+    public static String getSegmentId(String dataFileAbsolutePath) {
+      // find segment id from last of data file path
+      String tempdataFileAbsolutePath = dataFileAbsolutePath.replace(
+              CarbonCommonConstants.WINDOWS_FILE_SEPARATOR, CarbonCommonConstants.FILE_SEPARATOR);
+      int endIndex = tempdataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
+      // + 1 for size of "/"
+      int startIndex = tempdataFileAbsolutePath.lastIndexOf(
+              CarbonCommonConstants.FILE_SEPARATOR, endIndex - 1) + 1;
+      String segmentDirStr = dataFileAbsolutePath.substring(startIndex, endIndex);
+      //identify id in segment_<id>
+      String[] segmentDirSplits = segmentDirStr.split("_");
+      try {
+        if (segmentDirSplits.length == 2) {
+          return segmentDirSplits[1];
+        }
+      } catch (Exception e) {
+        return CarbonCommonConstants.INVALID_SEGMENT_ID;
+      }
+      return CarbonCommonConstants.INVALID_SEGMENT_ID;
+    }
+  }
+
+  /**
+   * Below method will be used to get sort index file present in mentioned folder
+   *
+   * @param sortIndexDir directory where sort index file resides
+   * @param columnUniqueId   columnunique id
+   * @return sort index carbon files
+   */
+  public CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir, final String columnUniqueId) {
+    CarbonFile[] files = sortIndexDir.listFiles(new CarbonFileFilter() {
+      @Override public boolean accept(CarbonFile file) {
+        return file.getName().startsWith(columnUniqueId) && file.getName().endsWith(SORT_INDEX_EXT);
+      }
+    });
+    return files;
+  }
+
+  /**
+   * returns the carbondata file name
+   *
+   * @param carbonDataFilePath carbondata file path
+   * @return
+   */
+  public static String getCarbonDataFileName(String carbonDataFilePath) {
+    String carbonDataFileName = carbonDataFilePath
+        .substring(carbonDataFilePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1,
+            carbonDataFilePath.indexOf(CARBON_DATA_EXT));
+    return carbonDataFileName;
+  }
+
+  /**
+   *
+   * @return carbon data extension
+   */
+  public static String getCarbonDataExtension() {
+    return CARBON_DATA_EXT;
+  }
+
+  /**
+   *
+   * @return carbon index extension
+   */
+  public static String getCarbonIndexExtension() {
+    return INDEX_FILE_EXT;
+  }
+
+  /**
+   * This method will remove strings in path and return short block id
+   *
+   * @param blockId
+   * @return shortBlockId
+   */
+  public static String getShortBlockId(String blockId) {
+    return blockId.replace(PARTITION_PREFIX, "")
+            .replace(SEGMENT_PREFIX, "")
+            .replace(DATA_PART_PREFIX, "")
+            .replace(CARBON_DATA_EXT, "");
+  }
+
+  /**
+   * This method will append strings in path and return block id
+   *
+   * @param shortBlockId
+   * @return blockId
+   */
+  public static String getBlockId(String shortBlockId) {
+    String[] splitRecords = shortBlockId.split(CarbonCommonConstants.FILE_SEPARATOR);
+    StringBuffer sb = new StringBuffer();
+    for (int i = 0; i < splitRecords.length; i++) {
+      if (i == 0) {
+        sb.append(PARTITION_PREFIX);
+        sb.append(splitRecords[i]);
+      } else if (i == 1) {
+        sb.append(CarbonCommonConstants.FILE_SEPARATOR);
+        sb.append(SEGMENT_PREFIX);
+        sb.append(splitRecords[i]);
+      } else if (i == 2) {
+        sb.append(CarbonCommonConstants.FILE_SEPARATOR);
+        sb.append(DATA_PART_PREFIX);
+        sb.append(splitRecords[i]);
+      } else if (i == 3) {
+        sb.append(CarbonCommonConstants.FILE_SEPARATOR);
+        sb.append(splitRecords[i]);
+        sb.append(CARBON_DATA_EXT);
+      } else {
+        sb.append(CarbonCommonConstants.FILE_SEPARATOR);
+        sb.append(splitRecords[i]);
+      }
+    }
+    return sb.toString();
+  }
+
+
+  /**
+   * adds data part prefix to given value
+   * @return partition prefix
+   */
+  public static String addDataPartPrefix(String value) {
+    return DATA_PART_PREFIX + value;
+  }
+
+  /**
+   * adds part prefix to given value
+   * @return partition prefix
+   */
+  public static String addPartPrefix(String value) {
+    return PARTITION_PREFIX + value;
+  }
+
+  /**
+   * adds part prefix to given value
+   * @return partition prefix
+   */
+  public static String addSegmentPrefix(String value) {
+    return SEGMENT_PREFIX + value;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java
index 8f3b4d8..77867cb 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriter.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.writer;
 
 import java.io.IOException;
 
-import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
+import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails;
 
 /**
  * Delete delta file writer interface



Mime
View raw message