public CarbonMultiBlockSplit(List<CarbonInputSplit> splitList, String[] locations) { this.splitList = splitList; this.locations = locations; calculateLength(); }
protected FileSplit makeSplit(String segmentId, Path file, long start, long length, String[] hosts, FileFormat fileFormat) { return new CarbonInputSplit(segmentId, file, start, length, hosts, fileFormat); }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (!(obj instanceof CarbonInputSplit)) { return false; } CarbonInputSplit other = (CarbonInputSplit) obj; return 0 == this.compareTo(other); }
public QueryModel createQueryModel(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { Configuration configuration = taskAttemptContext.getConfiguration(); CarbonTable carbonTable = getOrCreateCarbonTable(configuration); String projectionString = getColumnProjection(configuration); String[] projectColumns; if (projectionString != null) { .filterExpression(getFilterPredicates(configuration)) .dataConverter(getDataTypeConverter(configuration)) .build(); List<String> invalidSegments = split.getAllSplits().get(0).getInvalidSegments(); if (invalidSegments.size() > 0) { queryModel.setInvalidSegmentIds(invalidSegments); split.getAllSplits().get(0).getInvalidTimestampRange(); if ((null != invalidTimestampRangeList) && (invalidTimestampRangeList.size() > 0)) { queryModel.setInvalidBlockForSegmentId(invalidTimestampRangeList);
@Override public T getCurrentValue() throws IOException, InterruptedException { rowCount += 1; if (null != inputMetricsStats) { inputMetricsStats.incrementRecordRead(1L); } return readSupport.readRow(carbonIterator.next()); }
@Override public Object getCurrentValue() throws IOException, InterruptedException { if (isVectorReader) { int value = vectorProxy.numRows(); if (inputMetricsStats != null) { inputMetricsStats.incrementRecordRead((long) value); } return vectorProxy.getColumnarBatch(); } if (inputMetricsStats != null) { inputMetricsStats.incrementRecordRead(1L); } return outputRow; }
@Override public RecordReader<Void, T> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { Configuration configuration = taskAttemptContext.getConfiguration(); QueryModel queryModel = createQueryModel(inputSplit, taskAttemptContext); CarbonReadSupport<T> readSupport = getReadSupportClass(configuration); return new CarbonRecordReader<T>(queryModel, readSupport, taskAttemptContext.getConfiguration()); }
/** * Method to check whether there exists any block which does not contain the blocklet info * * @param splitList * @return */ public static boolean isBlockWithoutBlockletInfoExists(List<CarbonInputSplit> splitList) { for (CarbonInputSplit inputSplit : splitList) { if (null == inputSplit.getDetailInfo().getBlockletInfo()) { return true; } } return false; }
/** * Set the column projection column names from CarbonProjection * * @param configuration Configuration info * @param projection CarbonProjection object that includes unique projection column name */ public static void setColumnProjection(Configuration configuration, CarbonProjection projection) { if (projection == null || projection.isEmpty()) { return; } String[] allColumns = projection.getAllColumns(); StringBuilder builder = new StringBuilder(); for (String column : allColumns) { builder.append(column).append(","); } String columnString = builder.toString(); columnString = columnString.substring(0, columnString.length() - 1); configuration.set(COLUMN_PROJECTION, columnString); }
@Override public void readFields(DataInput in) throws IOException { // read all splits int numSplit = in.readInt(); splitList = new ArrayList<>(numSplit); for (int i = 0; i < numSplit; i++) { CarbonInputSplit split = new CarbonInputSplit(); split.readFields(in); splitList.add(split); } int len = in.readInt(); locations = new String[len]; for (int i = 0; i < len; i++) { locations[i] = in.readUTF(); } fileFormat = FileFormat.getByOrdinal(in.readInt()); }
private CarbonInputSplit convertToCarbonInputSplit(ExtendedBlocklet blocklet) throws IOException { CarbonInputSplit split = CarbonInputSplit .from(blocklet.getSegmentId(), blocklet.getBlockletId(), new FileSplit(new Path(blocklet.getPath()), 0, blocklet.getLength(), blocklet.getLocations()), ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()), blocklet.getDataMapWriterPath()); split.setDetailInfo(blocklet.getDetailInfo()); return split; }
@Override public String getBlockPath() { return getPath().getName(); }
@Override public void write(DataOutput out) throws IOException { // write number of splits and then write all splits out.writeInt(splitList.size()); for (CarbonInputSplit split: splitList) { split.write(out); } out.writeInt(locations.length); for (int i = 0; i < locations.length; i++) { out.writeUTF(locations[i]); } out.writeInt(fileFormat.ordinal()); }
/** * get batch result * * @return rows */ public List<Object[]> getBatchValue() { if (null != inputMetricsStats) { inputMetricsStats.incrementRecordRead(1L); } List<Object[]> objects = ((ChunkRowIterator) carbonIterator).nextBatch(); rowCount += objects.size(); return objects; }
protected FileSplit makeSplit(String segmentId, Path file, long start, long length, String[] hosts, String[] inMemoryHosts, FileFormat fileFormat) { return new CarbonInputSplit(segmentId, file, start, length, hosts, inMemoryHosts, fileFormat); }
public CarbonMultiBlockSplit(List<CarbonInputSplit> splitList, String[] locations, FileFormat fileFormat) { this.splitList = splitList; this.locations = locations; this.fileFormat = fileFormat; calculateLength(); }
@Override public Object getCurrentValue() throws IOException, InterruptedException { if (returnColumnarBatch) { int value = carbonColumnarBatch.getActualSize(); rowCount += value; if (inputMetricsStats != null) { inputMetricsStats.incrementRecordRead((long) value); } return vectorProxy.getColumnarBatch(); } rowCount += 1; return vectorProxy.getRow(batchIdx - 1); }
public static CarbonInputSplit from(String segmentId, String blockletId, FileSplit split, ColumnarFormatVersion version, String dataMapWritePath) throws IOException { return new CarbonInputSplit(segmentId, blockletId, split.getPath(), split.getStart(), split.getLength(), split.getLocations(), version, null, dataMapWritePath); }
public Blocklet makeBlocklet() { return new Blocklet(getPath().getName(), blockletId); } }