public TupleForwarder(IHyracksTaskContext ctx, IFrameWriter writer) throws HyracksDataException { this.frame = new VSizeFrame(ctx); this.writer = writer; this.appender = new FrameTupleAppender(frame); }
private int[] buildGroup(RunFileReader reader, ISpillableTable table, RunFileWriter[] runFileWriters) throws HyracksDataException { ExternalHashGroupBy groupBy = new ExternalHashGroupBy(this, table, runFileWriters, partialAggRecordDesc); reader.open(); try { VSizeFrame frame = new VSizeFrame(ctx); while (reader.nextFrame(frame)) { groupBy.insert(frame.getBuffer()); } groupBy.flushSpilledPartitions(); } finally { reader.close(); } return groupBy.getSpilledNumTuples(); }
public static void processFeedMessage(ByteBuffer input, VSizeFrame message, FrameTupleAccessor fta) throws HyracksDataException { // read the message and reduce the number of tuples fta.reset(input); int tc = fta.getTupleCount() - 1; int offset = fta.getTupleStartOffset(tc); int len = fta.getTupleLength(tc); int newSize = FrameHelper.calcAlignedFrameSizeToStore(1, len, message.getMinSize()); message.reset(); message.ensureFrameSize(newSize); message.getBuffer().clear(); message.getBuffer().put(input.array(), offset, len); message.getBuffer().flip(); IntSerDeUtils.putInt(input.array(), FrameHelper.getTupleCountOffset(input.capacity()), tc); }
private void writeRandomMessage(VSizeFrame frame, byte tag, int size) throws HyracksDataException { // We subtract 2, 1 for the tag, and one for the end offset Random random = new Random(); byte[] bytes = new byte[size - 2]; random.nextBytes(bytes); int frameSize = FrameHelper.calcAlignedFrameSizeToStore(1, size - 1, DEFAULT_FRAME_SIZE); frame.ensureFrameSize(frameSize); frame.getBuffer().clear(); frame.getBuffer().put(tag); frame.getBuffer().put(bytes); frame.getBuffer().flip(); }
new RecordTupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS); VSizeFrame frame = new VSizeFrame(ctx); VSizeFrame marker = new VSizeFrame(ctx); FrameTupleAppender tupleAppender = new FrameTupleAppender(frame); long markerId = 0L; for (int j = 0; j < NUM_OF_RECORDS; j++) { if (j % SNAPSHOT_SIZE == 0) { marker.reset(); marker.getBuffer().put(MessagingFrameTupleAppender.MARKER_MESSAGE); marker.getBuffer().putLong(markerId); marker.getBuffer().flip(); markerId++; TaskUtil.put(HyracksConstants.KEY_MESSAGE, marker, ctx);
private void merge(IFrameWriter writer, List<GeneratedRunFileReader> partialRuns) throws HyracksDataException { RunMergingFrameReader merger = new RunMergingFrameReader(ctx, partialRuns, inFrames, getSortFields(), comparators, nmkComputer, recordDesc, topK); int io = 0; merger.open(); try { while (merger.nextFrame(outputFrame)) { FrameUtils.flushFrame(outputFrame.getBuffer(), writer); io++; } } finally { merger.close(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Output " + io + " frames"); } } }
@Override public void resize(int frameSize) throws HyracksDataException { if (getFrameSize() != frameSize) { buffer = ctx.reallocateFrame(buffer, frameSize, false); } }
@Override public void ensureFrameSize(int newSize) throws HyracksDataException { if (newSize > getFrameSize()) { buffer = ctx.reallocateFrame(buffer, newSize, true); } }
public FrameDeserializingDataReader(IHyracksTaskContext ctx, IFrameReader frameReader, RecordDescriptor recordDescriptor) throws HyracksDataException { this.frame = new VSizeFrame(ctx); this.frameReader = frameReader; this.frameDeserializer = new FrameDeserializer(recordDescriptor); }
try { writer.open(); VSizeFrame frame = new VSizeFrame(this); while (reader.nextFrame(frame)) { if (aborted) { return; ByteBuffer buffer = frame.getBuffer(); writer.nextFrame(buffer); buffer.compact();
public synchronized ByteBuffer next() throws HyracksDataException { frame.reset(); if (totalReadCount == totalWriteCount) { return null; bis.read(frame.getBuffer().array(), 0, frame.getFrameSize()); byte frameCount = frame.getBuffer().array()[0]; if (frameCount > 1) { frame.ensureFrameSize(frame.getMinSize() * frameCount); bis.read(frame.getBuffer().array(), frame.getMinSize(), frame.getFrameSize() - frame.getMinSize()); return frame.getBuffer(); } catch (Exception e) { throw HyracksDataException.create(e);
protected void allocateFrames(int i) throws HyracksDataException { appenders[i].reset(new VSizeFrame(ctx), true); allocatedFrames[i] = true; }
new MToNPartitioningWithMessageConnectorDescriptor(connDescRegistry, partitionComputerFactory); IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE); VSizeFrame message = new VSizeFrame(ctx); VSizeFrame tempBuffer = new VSizeFrame(ctx); TaskUtil.put(HyracksConstants.KEY_MESSAGE, message, ctx); message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE); message.getBuffer().flip(); ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, VSizeFrame frame = new VSizeFrame(ctx); FrameTupleAppender appender = new FrameTupleAppender(frame); for (int count = 0; count < NUMBER_OF_CONSUMERS; count++) { appender.append(tuple); partitioner.nextFrame(frame.getBuffer()); partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null); Assert.assertEquals(partitionWriterFactory.getWriters().get(0).nextFrameCount(), 1);
public SerializingDataWriter(IHyracksTaskContext ctx, RecordDescriptor recordDescriptor, IFrameWriter frameWriter) throws HyracksDataException { tb = new ArrayTupleBuilder(recordDescriptor.getFieldCount()); this.recordDescriptor = recordDescriptor; this.frameWriter = frameWriter; tupleAppender = new FrameTupleAppender(new VSizeFrame(ctx)); open = false; }
new MToNPartitioningWithMessageConnectorDescriptor(connDescRegistry, partitionComputerFactory); IHyracksTaskContext ctx = TestUtils.create(DEFAULT_FRAME_SIZE); VSizeFrame message = new VSizeFrame(ctx); VSizeFrame tempBuffer = new VSizeFrame(ctx); TaskUtil.put(HyracksConstants.KEY_MESSAGE, message, ctx); message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE); message.getBuffer().flip(); ISerializerDeserializer<?>[] serdes = new ISerializerDeserializer<?>[] { Integer64SerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, MessagingFrameTupleAppender.getMessageType(tempBuffer)); message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.ACK_REQ_FEED_MESSAGE); message.getBuffer().flip(); partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null); for (TestFrameWriter writer : recipients) { message.getBuffer().clear(); message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE); message.getBuffer().flip(); partitioner.flush(ITracer.NONE, null, ITraceCategoryRegistry.CATEGORIES_NONE, null); for (TestFrameWriter writer : recipients) {
@Override public IKeyValueParser<LongWritable, Text> createKeyValueParser(final IHyracksTaskContext ctx) throws HyracksDataException { final ArrayTupleBuilder tb = new ArrayTupleBuilder(1); final FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx)); return new IKeyValueParser<LongWritable, Text>() { @Override public void open(IFrameWriter writer) { } @Override public void parse(LongWritable key, Text value, IFrameWriter writer, String fileString) throws HyracksDataException { tb.reset(); tb.addField(value.getBytes(), 0, value.getLength()); FrameUtils.appendToWriter(writer, appender, tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize()); } @Override public void close(IFrameWriter writer) throws HyracksDataException { appender.write(writer, false); } }; }
new Long(NUM_FRAMES * DEFAULT_FRAME_SIZE)); spiller.open(); VSizeFrame frame = new VSizeFrame(ctx); spiller.spill(frame.getBuffer()); Assert.assertEquals(1, spiller.remaining()); Assert.assertEquals(1, countSpillFiles()); for (int i = 0; i < 1022; i++) { spiller.spill(frame.getBuffer()); spiller.spill(frame.getBuffer()); Assert.assertEquals(1024, spiller.remaining()); Assert.assertEquals(2, countSpillFiles()); for (int i = 0; i < 1023; i++) { spiller.spill(frame.getBuffer()); spiller.spill(frame.getBuffer()); Assert.assertEquals(2048, spiller.remaining()); Assert.assertEquals(3, countSpillFiles());
public FrameSpiller(IHyracksTaskContext ctx, String fileNamePrefix, long budgetInBytes) throws HyracksDataException { this.frame = new VSizeFrame(ctx); this.fileNamePrefix = fileNamePrefix; this.budget = (int) Math.min(budgetInBytes / ctx.getInitialFrameSize(), Integer.MAX_VALUE); if (budget <= 0) { throw new HyracksDataException("Invalid budget " + budgetInBytes + ". Budget must be larger than 0"); } }
FeedRuntimeInputHandler handler = createInputHandler(ctx, writer, fpa, framePool); handler.open(); VSizeFrame frame = new VSizeFrame(ctx); handler.nextFrame(frame.getBuffer()); Future<?> result = EXECUTOR.submit(new Pusher(frame.getBuffer(), handler)); result.get();
@Override public IPushRuntime[] createPushRuntime(final IHyracksTaskContext ctx) throws HyracksDataException { return new IPushRuntime[] { new AbstractOneInputSourcePushRuntime() { private final ArrayTupleBuilder tb = new ArrayTupleBuilder(0); private final FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx)); @Override public void open() throws HyracksDataException { super.open(); if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) { throw new IllegalStateException(); } appender.write(writer, true); } @Override public void close() throws HyracksDataException { if (isOpen) { writer.close(); } } @Override public void flush() throws HyracksDataException { appender.flush(writer); } } }; } }