@Override public void close() throws IOException { this.parquetRecordReader.close(); }
@Override public boolean next(Void key, Container<V> value) throws IOException { if (eof) { return false; } if (firstRecord) { // key & value are already read. firstRecord = false; return true; } try { if (realReader.nextKeyValue()) { if (value != null) value.set(realReader.getCurrentValue()); return true; } } catch (InterruptedException e) { throw new IOException(e); } eof = true; // strictly not required, just for consistency return false; } }
@Override public float getProgress() throws IOException { try { return realReader.getProgress(); } catch (InterruptedException e) { Thread.interrupted(); throw new IOException(e); } }
public void initialize(InputSplit inputSplit, Configuration configuration, Reporter reporter) throws IOException, InterruptedException { BenchmarkCounter.initCounterFromReporter(reporter,configuration); initializeInternalReader(toParquetSplit(inputSplit), configuration); }
public RecordReaderWrapper( InputSplit oldSplit, JobConf oldJobConf, Reporter reporter) throws IOException { splitLen = oldSplit.getLength(); try { realReader = new ParquetRecordReader<V>( ParquetInputFormat.<V>getReadSupportInstance(oldJobConf), ParquetInputFormat.getFilter(oldJobConf)); if (oldSplit instanceof ParquetInputSplitWrapper) { realReader.initialize(((ParquetInputSplitWrapper) oldSplit).realSplit, oldJobConf, reporter); } else if (oldSplit instanceof FileSplit) { realReader.initialize((FileSplit) oldSplit, oldJobConf, reporter); } else { throw new IllegalArgumentException( "Invalid split (not a FileSplit or ParquetInputSplitWrapper): " + oldSplit); } // read once to gain access to key and value objects if (realReader.nextKeyValue()) { firstRecord = true; valueContainer = new Container<V>(); valueContainer.set(realReader.getCurrentValue()); } else { eof = true; } } catch (InterruptedException e) { Thread.interrupted(); throw new IOException(e); } }
@Override public void initialize(GuaguaFileSplit split) throws IOException { ReadSupport<Tuple> readSupport = getReadSupportInstance(this.conf); this.parquetRecordReader = new ParquetRecordReader<Tuple>(readSupport, getFilter(this.conf)); ParquetInputSplit parquetInputSplit = new ParquetInputSplit(new Path(split.getPath()), split.getOffset(), split.getOffset() + split.getLength(), split.getLength(), null, null); try { this.parquetRecordReader.initialize(parquetInputSplit, buildContext()); } catch (InterruptedException e) { throw new GuaguaRuntimeException(e); } }
@Override public boolean nextKeyValue() throws IOException { try { return this.parquetRecordReader.nextKeyValue(); } catch (InterruptedException e) { throw new GuaguaRuntimeException(e); } }
realReader.initialize(split, taskContext); return realReader;
/** * {@inheritDoc} */ @Override public RecordReader<Void, T> createRecordReader( InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { Configuration conf = ContextUtil.getConfiguration(taskAttemptContext); ReadSupport<T> readSupport = getReadSupport(conf); return new ParquetRecordReader<T>(readSupport, getFilter(conf)); }
@Override public GuaguaWritableAdapter<Tuple> getCurrentValue() { try { return new GuaguaWritableAdapter<Tuple>(this.parquetRecordReader.getCurrentValue()); } catch (IOException e) { throw new GuaguaRuntimeException(e); } catch (InterruptedException e) { throw new GuaguaRuntimeException(e); } }
@Override public boolean advanceNextPosition() { try { // reset null flags System.arraycopy(nullsRowDefault, 0, nulls, 0, isPartitionColumn.length); if (closed || !recordReader.nextKeyValue()) { close(); return false; } return true; } catch (IOException | RuntimeException | InterruptedException e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } closeWithSuppression(e); throw new PrestoException(HIVE_CURSOR_ERROR, e); } }
/** * {@inheritDoc} */ @Override public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException, InterruptedException { if (context instanceof TaskInputOutputContext<?, ?, ?, ?>) { BenchmarkCounter.initCounterFromContext((TaskInputOutputContext<?, ?, ?, ?>) context); } else { LOG.error("Can not initialize counter due to context is not a instance of TaskInputOutputContext, but is " + context.getClass().getCanonicalName()); } initializeInternalReader(toParquetSplit(inputSplit), ContextUtil.getConfiguration(context)); }
@Override public void close() throws IOException { realReader.close(); }
private void updateCompletedBytes() { try { long newCompletedBytes = (long) (totalBytes * recordReader.getProgress()); completedBytes = min(totalBytes, max(completedBytes, newCompletedBytes)); } catch (IOException ignored) { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } }
@Override public void close() { // some hive input formats are broken and bad things can happen if you close them multiple times if (closed) { return; } closed = true; updateCompletedBytes(); try { recordReader.close(); } catch (IOException e) { throw Throwables.propagate(e); } }