@Override public SplitReader<K, V> createSplitReader(Split split) { return new ForwardingSplitReader<K, V>(delegate.createSplitReader(split)) { @Override public void close() { try { try { super.close(); } finally { flushOperations(); } } catch (Exception e) { throw Throwables.propagate(e); } } }; } };
@Override public List<Split> getSplits() { try { try { return delegate.getSplits(); } finally { flushOperations(); } } catch (Exception e) { throw Throwables.propagate(e); } }
@Override public List<Split> getSplits() { try { try { return delegate.getSplits(); } finally { flushOperations(); } } catch (Exception e) { throw Throwables.propagate(e); } }
private Map<String, String> createBatchReadableConfiguration() { List<Split> splits = this.splits; if (splits == null) { splits = ((BatchReadable<?, ?>) dataset).getSplits(); } Configuration hConf = new Configuration(); hConf.clear(); try { AbstractBatchReadableInputFormat.setDatasetSplits(hConf, datasetNamespace, datasetName, datasetArgs, splits); return ConfigurationUtil.toMap(hConf); } catch (IOException e) { throw new IllegalArgumentException(e); } } }
@Override public SplitReader<K, V> createSplitReader(Split split) { return new ForwardingSplitReader<K, V>(delegate.createSplitReader(split)) { @Override public void close() { try { try { super.close(); } finally { flushOperations(); } } catch (Exception e) { throw Throwables.propagate(e); } } }; } };
private Map<String, String> createBatchReadableConfiguration() { List<Split> splits = this.splits; if (splits == null) { splits = ((BatchReadable<?, ?>) dataset).getSplits(); } Configuration hConf = new Configuration(); hConf.clear(); try { AbstractBatchReadableInputFormat.setDatasetSplits(hConf, datasetNamespace, datasetName, datasetArgs, splits); return ConfigurationUtil.toMap(hConf); } catch (IOException e) { throw new IllegalArgumentException(e); } } }
@Override public RecordReader<KEY, VALUE> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { DataSetInputSplit inputSplit = (DataSetInputSplit) split; Configuration conf = context.getConfiguration(); String datasetNamespace = conf.get(DATASET_NAMESPACE); String datasetName = conf.get(DATASET_NAME); Map<String, String> datasetArgs = GSON.fromJson(conf.get(DATASET_ARGS), DATASET_ARGS_TYPE); @SuppressWarnings("unchecked") BatchReadable<KEY, VALUE> batchReadable = createBatchReadable(context, datasetNamespace, datasetName, datasetArgs); SplitReader<KEY, VALUE> splitReader = batchReadable.createSplitReader(inputSplit.getSplit()); return new SplitReaderRecordReader<>(splitReader); }
@Override public RecordReader<KEY, VALUE> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { DataSetInputSplit inputSplit = (DataSetInputSplit) split; Configuration conf = context.getConfiguration(); String datasetNamespace = conf.get(DATASET_NAMESPACE); String datasetName = conf.get(DATASET_NAME); Map<String, String> datasetArgs = GSON.fromJson(conf.get(DATASET_ARGS), DATASET_ARGS_TYPE); @SuppressWarnings("unchecked") BatchReadable<KEY, VALUE> batchReadable = createBatchReadable(context, datasetNamespace, datasetName, datasetArgs); SplitReader<KEY, VALUE> splitReader = batchReadable.createSplitReader(inputSplit.getSplit()); return new SplitReaderRecordReader<>(splitReader); }