@Override public boolean reachedEnd() throws IOException { return !dataFileReader.hasNext() || dataFileReader.pastSync(end); }
/** {@inheritDoc} */ @Override public boolean nextKeyValue() throws IOException, InterruptedException { assert null != mAvroFileReader; if (mAvroFileReader.hasNext() && !mAvroFileReader.pastSync(mEndPosition)) { mCurrentRecord = mAvroFileReader.next(mCurrentRecord); return true; } return false; }
public boolean next(TetherData data, NullWritable ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; data.buffer(reader.nextBlock()); data.count((int)reader.getBlockCount()); return true; }
public void testSplits() throws IOException { File file = makeFile(); DataFileReader<Object> reader = new DataFileReader<>(file, new GenericDatumReader<>()); Random rand = new Random(SEED); try { int splits = 10; // number of splits int length = (int)file.length(); // length of file int end = length; // end of split int remaining = end; // bytes remaining int count = 0; // count of entries while (remaining > 0) { int start = Math.max(0, end - rand.nextInt(2*length/splits)); reader.sync(start); // count entries in split while (!reader.pastSync(end)) { reader.next(); count++; } remaining -= end-start; end = start; } assertEquals(COUNT, count); } finally { reader.close(); } }
@Override public boolean reachedEnd() throws IOException { return !dataFileReader.hasNext() || dataFileReader.pastSync(end); }
@Override public boolean reachedEnd() throws IOException { return !dataFileReader.hasNext() || dataFileReader.pastSync(end); }
@Override public boolean reachedEnd() throws IOException { return !dataFileReader.hasNext() || dataFileReader.pastSync(end); }
@Override public boolean reachedEnd() throws IOException { return !dataFileReader.hasNext() || dataFileReader.pastSync(end); }
/** {@inheritDoc} */ @Override public boolean nextKeyValue() throws IOException, InterruptedException { assert null != mAvroFileReader; if (mAvroFileReader.hasNext() && !mAvroFileReader.pastSync(mEndPosition)) { mCurrentRecord = mAvroFileReader.next(mCurrentRecord); return true; } return false; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!reader.hasNext() || reader.pastSync(end)) { return false; } datum = reader.next(datum); return true; }
@Override public boolean nextInner() throws IOException { if (!reader.hasNext()) return false; if(end > 0 && reader.pastSync(end)) return false; persistent = reader.next(persistent); return true; }
@Override public boolean nextInner() throws IOException { if (!reader.hasNext()) return false; if(end > 0 && reader.pastSync(end)) return false; persistent = reader.next(persistent); return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!reader.hasNext() || reader.pastSync(end)) { return false; } pair = Convert2Pair(reader.next(reuse)); return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!reader.hasNext() || reader.pastSync(end)) { return false; } pair = Convert2Pair(reader.next(reuse)); return true; }
public boolean next(AvroWrapper<T> wrapper, NullWritable ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; wrapper.datum(reader.next(wrapper.datum())); return true; }
@Override public int next() { final Stopwatch watch = Stopwatch.createStarted(); if (reader == null) { throw new IllegalStateException("Avro reader is not open."); } if (!reader.hasNext()) { return 0; } int recordCount = 0; writer.allocate(); writer.reset(); try { for (GenericContainer container = null; recordCount < numRowsPerBatch && reader.hasNext() && !reader.pastSync(end); recordCount++) { writer.setPosition(recordCount); container = reader.next(container); processRecord(container, container.getSchema()); } writer.setValueCount(recordCount); } catch (IOException e) { throw new RuntimeException(e); } logger.debug("Read {} records in {} ms", recordCount, watch.elapsed(TimeUnit.MILLISECONDS)); return recordCount; }
public boolean next(TetherData data, NullWritable ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; data.buffer(reader.nextBlock()); data.count((int)reader.getBlockCount()); return true; }
public boolean next(TetherData data, NullWritable ignore) throws IOException { if (!reader.hasNext() || reader.pastSync(end)) return false; data.buffer(reader.nextBlock()); data.count((int)reader.getBlockCount()); return true; }
@Override public int next() { final Stopwatch watch = Stopwatch.createStarted(); if (reader == null) { throw new IllegalStateException("Avro reader is not open."); } if (!reader.hasNext()) { return 0; } int recordCount = 0; writer.allocate(); writer.reset(); try { for (GenericContainer container = null; recordCount < DEFAULT_BATCH_SIZE && reader.hasNext() && !reader.pastSync(end); recordCount++) { writer.setPosition(recordCount); container = reader.next(container); processRecord(container, container.getSchema()); } writer.setValueCount(recordCount); } catch (IOException e) { throw new DrillRuntimeException(e); } logger.debug("Read {} records in {} ms", recordCount, watch.elapsed(TimeUnit.MILLISECONDS)); return recordCount; }