throws IOException { List<ColumnEncoding> allEnc = footer.getColumnsList(); OrcProto.StripeInformation si = dirEntry.build(); if (LlapIoImpl.LOG.isTraceEnabled()) { LlapIoImpl.LOG.trace(("Finalizing stripe " + footer.build() + " => " + si)
private List<StripeInformation> createStripes(long... rowCounts) { long offset = 0; List<StripeInformation> result = new ArrayList<StripeInformation>(rowCounts.length); for(long count: rowCounts) { OrcProto.StripeInformation.Builder stripe = OrcProto.StripeInformation.newBuilder(); stripe.setDataLength(800).setIndexLength(100).setFooterLength(100) .setNumberOfRows(count).setOffset(offset); offset += 1000; result.add(new ReaderImpl.StripeInformationImpl(stripe.build())); } return result; }
OrcProto.StripeInformation dirEntry = OrcProto.StripeInformation.newBuilder() .setOffset(start) .setNumberOfRows(rowsInStripe) .setIndexLength(indexSize) .setDataLength(dataSize) .setFooterLength(footerLength).build(); stripes.add(dirEntry); rowCount += rowsInStripe;
OrcProto.StripeInformation dirEntry = OrcProto.StripeInformation.newBuilder() .setOffset(start) .setNumberOfRows(rowsInStripe) .setIndexLength(indexSize) .setDataLength(dataSize) .setFooterLength(footerLength).build(); stripes.add(dirEntry); rowCount += rowsInStripe;
throws IOException { List<ColumnEncoding> allEnc = footer.getColumnsList(); OrcProto.StripeInformation si = dirEntry.build(); if (LlapIoImpl.LOG.isTraceEnabled()) { LlapIoImpl.LOG.trace(("Finalizing stripe " + footer.build() + " => " + si)
.setOffset(start) .setNumberOfRows(rowsInStripe) .setIndexLength(stripeInfo.getIndexLength()) .setDataLength(stripeInfo.getDataLength()) .setFooterLength(stripeInfo.getFooterLength()) .build(); stripes.add(dirEntry);
.setOffset(start) .setNumberOfRows(rowsInStripe) .setIndexLength(stripeInfo.getIndexLength()) .setDataLength(stripeInfo.getDataLength()) .setFooterLength(stripeInfo.getFooterLength()) .build(); stripes.add(dirEntry);
.setNumberOfRows(rowsInStripe); physicalWriter.finalizeStripe(builder, dirEntry); stripes.add(dirEntry.build()); rowCount += rowsInStripe; rowsInStripe = 0;
@Override public void appendRawStripe(ByteBuffer buffer, OrcProto.StripeInformation.Builder dirEntry) throws IOException { long start = rawWriter.getPos(); int length = buffer.remaining(); long availBlockSpace = blockSize - (start % blockSize); // see if stripe can fit in the current hdfs block, else pad the remaining // space in the block if (length < blockSize && length > availBlockSpace && addBlockPadding) { byte[] pad = new byte[(int) Math.min(HDFS_BUFFER_SIZE, availBlockSpace)]; LOG.info(String.format("Padding ORC by %d bytes while merging..", availBlockSpace)); start += availBlockSpace; while (availBlockSpace > 0) { int writeLen = (int) Math.min(availBlockSpace, pad.length); rawWriter.write(pad, 0, writeLen); availBlockSpace -= writeLen; } } rawWriter.write(buffer.array(), buffer.arrayOffset() + buffer.position(), length); dirEntry.setOffset(start); }
dirEntry.setIndexLength(indexSize).setDataLength(dataSize);
@Override public void appendStripe(byte[] stripe, int offset, int length, StripeInformation stripeInfo, OrcProto.StripeStatistics stripeStatistics) throws IOException { checkArgument(stripe != null, "Stripe must not be null"); checkArgument(length <= stripe.length, "Specified length must not be greater specified array length"); checkArgument(stripeInfo != null, "Stripe information must not be null"); checkArgument(stripeStatistics != null, "Stripe statistics must not be null"); rowsInStripe = stripeInfo.getNumberOfRows(); // update stripe information OrcProto.StripeInformation.Builder dirEntry = OrcProto.StripeInformation .newBuilder() .setNumberOfRows(rowsInStripe) .setIndexLength(stripeInfo.getIndexLength()) .setDataLength(stripeInfo.getDataLength()) .setFooterLength(stripeInfo.getFooterLength()); physicalWriter.appendRawStripe(ByteBuffer.wrap(stripe, offset, length), dirEntry); // since we have already written the stripe, just update stripe statistics treeWriter.updateFileStatistics(stripeStatistics); fileMetadata.addStripeStats(stripeStatistics); stripes.add(dirEntry.build()); // reset it after writing the stripe rowCount += rowsInStripe; rowsInStripe = 0; }
public Builder mergeFrom(org.apache.orc.OrcProto.StripeInformation other) { if (other == org.apache.orc.OrcProto.StripeInformation.getDefaultInstance()) return this; if (other.hasOffset()) { setOffset(other.getOffset()); } if (other.hasIndexLength()) { setIndexLength(other.getIndexLength()); } if (other.hasDataLength()) { setDataLength(other.getDataLength()); } if (other.hasFooterLength()) { setFooterLength(other.getFooterLength()); } if (other.hasNumberOfRows()) { setNumberOfRows(other.getNumberOfRows()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }
/** * <code>repeated .orc.proto.StripeInformation stripes = 3;</code> */ public Builder setStripes( int index, org.apache.orc.OrcProto.StripeInformation.Builder builderForValue) { if (stripesBuilder_ == null) { ensureStripesIsMutable(); stripes_.set(index, builderForValue.build()); onChanged(); } else { stripesBuilder_.setMessage(index, builderForValue.build()); } return this; } /**
/** * <code>repeated .orc.proto.StripeInformation stripes = 3;</code> */ public Builder addStripes( org.apache.orc.OrcProto.StripeInformation.Builder builderForValue) { if (stripesBuilder_ == null) { ensureStripesIsMutable(); stripes_.add(builderForValue.build()); onChanged(); } else { stripesBuilder_.addMessage(builderForValue.build()); } return this; } /**
/** * <code>repeated .orc.proto.StripeInformation stripes = 3;</code> */ public Builder addStripes( int index, org.apache.orc.OrcProto.StripeInformation.Builder builderForValue) { if (stripesBuilder_ == null) { ensureStripesIsMutable(); stripes_.add(index, builderForValue.build()); onChanged(); } else { stripesBuilder_.addMessage(index, builderForValue.build()); } return this; } /**
private static Builder create() { return new Builder(); }
private void writeStripeFooter(OrcProto.StripeFooter footer, long dataSize, long indexSize, OrcProto.StripeInformation.Builder dirEntry) throws IOException { footer.writeTo(protobufWriter); protobufWriter.flush(); writer.flush(); dirEntry.setOffset(stripeStart); dirEntry.setFooterLength(rawWriter.getPos() - stripeStart - dataSize - indexSize); }
@java.lang.Override protected Builder newBuilderForType( com.google.protobuf.GeneratedMessage.BuilderParent parent) { Builder builder = new Builder(parent); return builder; } /**
for(long stripeLength: stripeLengths) { footer.addStripes(OrcProto.StripeInformation.newBuilder() .setOffset(offset) .setIndexLength(0) .setDataLength(stripeLength-10) .setFooterLength(10) .setNumberOfRows(1000)); offset += stripeLength;