@Override public void close() throws IOException { try { recordScanner.close(); } finally { recordScannable.close(); datasetAccessor.close(); } }
private void initialize() throws IOException { try { datasetAccessor.initialize(); recordScannable = datasetAccessor.getDataset(); recordScanner = recordScannable.createSplitRecordScanner( new Split() { @Override public long getLength() { return datasetInputSplit.getLength(); } } ); recordScanner.initialize(datasetInputSplit.getDataSetSplit()); initialized.set(true); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new IOException("Interrupted while initializing reader", ie); } catch (Exception e) { throw new IOException("Unable to get dataset", e); } }
@Override public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException { try (DatasetAccessor datasetAccessor = new DatasetAccessor(jobConf)) { try { datasetAccessor.initialize(); } catch (Exception e) { throw new IOException("Could not get dataset", e); } try (RecordScannable recordScannable = datasetAccessor.getDataset()) { Job job = new Job(jobConf); JobContext jobContext = ShimLoader.getHadoopShims().newJobContext(job); Path[] tablePaths = FileInputFormat.getInputPaths(jobContext); List<Split> dsSplits = recordScannable.getSplits(); InputSplit[] inputSplits = new InputSplit[dsSplits.size()]; for (int i = 0; i < dsSplits.size(); i++) { inputSplits[i] = new DatasetInputSplit(dsSplits.get(i), tablePaths[0]); } return inputSplits; } } }
if (isRecordScannable || isRecordWritable) { Type recordType = isRecordScannable ? ((RecordScannable) dataset).getRecordType() : ((RecordWritable) dataset).getRecordType();
if (isRecordScannable || isRecordWritable) { Type recordType = isRecordScannable ? ((RecordScannable) dataset).getRecordType() : ((RecordWritable) dataset).getRecordType();
recordType = ((RecordScannable) dataset).getRecordType(); } else if (dataset instanceof RecordWritable) { recordType = ((RecordWritable) dataset).getRecordType();
@Override public void close() throws IOException { try { recordScanner.close(); } finally { recordScannable.close(); datasetAccessor.close(); } }
private void initialize() throws IOException { try { datasetAccessor.initialize(); recordScannable = datasetAccessor.getDataset(); recordScanner = recordScannable.createSplitRecordScanner( new Split() { @Override public long getLength() { return datasetInputSplit.getLength(); } } ); recordScanner.initialize(datasetInputSplit.getDataSetSplit()); initialized.set(true); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new IOException("Interrupted while initializing reader", ie); } catch (Exception e) { throw new IOException("Unable to get dataset", e); } }
@Override public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException { try (DatasetAccessor datasetAccessor = new DatasetAccessor(jobConf)) { try { datasetAccessor.initialize(); } catch (Exception e) { throw new IOException("Could not get dataset", e); } try (RecordScannable recordScannable = datasetAccessor.getDataset()) { Job job = new Job(jobConf); JobContext jobContext = ShimLoader.getHadoopShims().newJobContext(job); Path[] tablePaths = FileInputFormat.getInputPaths(jobContext); List<Split> dsSplits = recordScannable.getSplits(); InputSplit[] inputSplits = new InputSplit[dsSplits.size()]; for (int i = 0; i < dsSplits.size(); i++) { inputSplits[i] = new DatasetInputSplit(dsSplits.get(i), tablePaths[0]); } return inputSplits; } } }
recordType = ((RecordScannable) dataset).getRecordType(); } else if (dataset instanceof RecordWritable) { recordType = ((RecordWritable) dataset).getRecordType();