@Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; PartitionSpec spec = (PartitionSpec) o; return Objects.equals(getLocation(), spec.getLocation()); }
/** * Returs the current partition specs of this segment * @return */ public List<PartitionSpec> getPartitionSpecs() { List<PartitionSpec> partitionSpecs = new ArrayList<>(); if (segmentFile != null) { for (Map.Entry<String, FolderDetails> entry : segmentFile.getLocationMap().entrySet()) { String location = entry.getKey(); if (entry.getValue().isRelative) { location = tablePath + CarbonCommonConstants.FILE_SEPARATOR + location; } if (entry.getValue().getStatus().equals(SegmentStatus.SUCCESS.getMessage())) { partitionSpecs.add(new PartitionSpec(entry.getValue().partitions, location)); } } } return partitionSpecs; }
private boolean isCorrectUUID(String[] fileDetails, PartitionSpec spec) { boolean needToScan = false; if (spec.getUuid() != null) { String[] split = spec.getUuid().split("_"); if (split[0].equals(fileDetails[2]) && CarbonTablePath.DataFileUtil .getTimeStampFromFileName(fileDetails[1]).equals(split[1])) { needToScan = true; } } else { needToScan = true; } return needToScan; }
SegmentFile segmentFile = null; for (PartitionSpec spec : partitionSpecs) { String location = spec.getLocation().toString(); CarbonFile carbonFile = FileFactory.getCarbonFile(location); FolderDetails folderDetails = new FolderDetails(); folderDetails.setRelative(isRelative); folderDetails.setPartitions(spec.getPartitions()); folderDetails.setStatus(SegmentStatus.SUCCESS.getMessage()); for (CarbonFile file : listFiles) {
/** * Get the partition specs of the segment * @param segmentId * @param tablePath * @return * @throws IOException */ public static List<PartitionSpec> getPartitionSpecs(String segmentId, String tablePath) throws IOException { LoadMetadataDetails segEntry = null; LoadMetadataDetails[] details = SegmentStatusManager.readLoadMetadata(CarbonTablePath.getMetadataPath(tablePath)); for (LoadMetadataDetails entry : details) { if (entry.getLoadName().equals(segmentId)) { segEntry = entry; break; } } if (segEntry != null && segEntry.getSegmentFile() != null) { SegmentFileStore fileStore = new SegmentFileStore(tablePath, segEntry.getSegmentFile()); List<PartitionSpec> partitionSpecs = fileStore.getPartitionSpecs(); for (PartitionSpec spec : partitionSpecs) { spec.setUuid(segmentId + "_" + segEntry.getLoadStartTime()); } return partitionSpecs; } return null; }
SegmentFileStore .writeSegmentFile(carbonLoadModel.getTablePath(), carbonLoadModel.getTaskNo(), partitionSpec.getLocation().toString(), carbonLoadModel.getFactTimeStamp() + "", partitionSpec.getPartitions()); } catch (IOException e) { throw e;
partitionSpec.getLocation().toString(), loadModel.getFactTimeStamp() + "", partitionSpec.getPartitions());
private static boolean pathExistsInPartitionSpec(List<PartitionSpec> partitionSpecs, Path partitionPath) { for (PartitionSpec spec : partitionSpecs) { if (spec.getLocation().equals(partitionPath)) { return true; } } return false; }
private boolean validatePartitionInfo(List<PartitionSpec> partitions) { // First get the partitions which are stored inside datamap. String[] fileDetails = getFileDetails(); // Check the exact match of partition information inside the stored partitions. boolean found = false; Path folderPath = new Path(fileDetails[0]); for (PartitionSpec spec : partitions) { if (folderPath.equals(spec.getLocation()) && isCorrectUUID(fileDetails, spec)) { found = true; break; } } return found; }
if (path.equals(spec.getLocation())) { entry.getValue().setStatus(SegmentStatus.MARKED_FOR_DELETE.getMessage()); updateSegment = true;
/** * initialise carbon data writer instance */ private void initDataHandler() throws Exception { String carbonStoreLocation; if (partitionSpec != null) { carbonStoreLocation = partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR + carbonLoadModel.getFactTimeStamp() + ".tmp"; } else { carbonStoreLocation = CarbonDataProcessorUtil .createCarbonStoreLocation(carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable(), carbonLoadModel.getSegmentId()); } CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel .getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName, tempStoreLocation, carbonStoreLocation); carbonFactDataHandlerModel.setSegmentId(carbonLoadModel.getSegmentId()); setDataFileAttributesInModel(carbonLoadModel, compactionType, carbonFactDataHandlerModel); dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(carbonFactDataHandlerModel); try { dataHandler.initialise(); } catch (CarbonDataWriterException e) { LOGGER.error(e); throw new Exception("Problem initialising data handler during compaction: " + e.getMessage()); } }
public RowResultMergerProcessor(String databaseName, String tableName, SegmentProperties segProp, String[] tempStoreLocation, CarbonLoadModel loadModel, CompactionType compactionType, PartitionSpec partitionSpec) throws IOException { this.segprop = segProp; this.partitionSpec = partitionSpec; this.loadModel = loadModel; CarbonDataProcessorUtil.createLocations(tempStoreLocation); String carbonStoreLocation; if (partitionSpec != null) { carbonStoreLocation = partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR + loadModel .getFactTimeStamp() + ".tmp"; } else { carbonStoreLocation = CarbonDataProcessorUtil .createCarbonStoreLocation(loadModel.getCarbonDataLoadSchema().getCarbonTable(), loadModel.getSegmentId()); } CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel .getCarbonFactDataHandlerModel(loadModel, loadModel.getCarbonDataLoadSchema().getCarbonTable(), segProp, tableName, tempStoreLocation, carbonStoreLocation); setDataFileAttributesInModel(loadModel, compactionType, carbonFactDataHandlerModel); carbonFactDataHandlerModel.setCompactionFlow(true); carbonFactDataHandlerModel.setSegmentId(loadModel.getSegmentId()); this.noDicAndComplexColumns = carbonFactDataHandlerModel.getNoDictAndComplexColumns(); dataHandler = new CarbonFactDataHandlerColumnar(carbonFactDataHandlerModel); }