@Override public void close() throws Exception { format.close(); if (format instanceof RichInputFormat) { ((RichInputFormat) format).closeInputFormat(); } }
@Override public void writeRecord(T record) throws IOException { this.blockBasedOutput.startRecord(); this.serialize(record, outView); }
@Override public void openInputFormat() throws IOException { if (this.replicatedIF instanceof RichInputFormat) { ((RichInputFormat)this.replicatedIF).openInputFormat(); } }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); this.blockInfo = this.createAndReadBlockInfo(); // We set the size of the BlockBasedInput to splitLength as each split contains one block. // After reading the block info, we seek in the file to the correct position. this.readRecords = 0; this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockInfo.getFirstRecordStart(), this.splitLength); this.dataInputStream = new DataInputViewStreamWrapper(blockBasedInput); }
protected FileInputFormat(Path filePath) { if (filePath != null) { setFilePath(filePath); } }
@Override public void setRuntimeContext(RuntimeContext context) { if (this.replicatedIF instanceof RichInputFormat) { ((RichInputFormat)this.replicatedIF).setRuntimeContext(context); } }
@Override public void closeInputFormat() throws IOException { if (this.replicatedIF instanceof RichInputFormat) { ((RichInputFormat)this.replicatedIF).closeInputFormat(); } } }
@Override public void configure(Configuration parameters) { super.configure(parameters); // the if is to prevent the configure() method from // overwriting the value set by the setter if (this.blockSize == NATIVE_BLOCK_SIZE) { long blockSize = parameters.getLong(BLOCK_SIZE_PARAMETER_KEY, NATIVE_BLOCK_SIZE); setBlockSize(blockSize); } }
@Override protected FSDataInputStream decorateInputStream(FSDataInputStream inputStream, FileInputSplit fileSplit) throws Throwable { inputStream = super.decorateInputStream(inputStream, fileSplit); return new InputStreamFSInputWrapper(new InvertedInputStream(inputStream)); } }
@Override protected String getDirectoryFileName(int taskNumber) { if(testFileName) { return "fancy-"+(taskNumber+1)+"-"+taskNumber+".avro"; } else { return super.getDirectoryFileName(taskNumber); } } }
@Override public T nextRecord(T record) throws IOException { if (this.reachedEnd()) { return null; } record = this.deserialize(record, this.dataInputStream); this.readRecords++; return record; }
@Override public OT nextRecord(OT record) throws IOException { if (readLine()) { return readRecord(record, this.currBuffer, this.currOffset, this.currLen); } else { this.end = true; return null; } }
private ArrayList<PathMatcher> getIncludeMatchers() { if (includeMatchers == null) { includeMatchers = buildPatterns(includePatterns); } return includeMatchers; }
@Override public int read(byte[] b) throws IOException { return read(b, 0, b.length); } }
/** * Closes the input by releasing all buffers and closing the file input stream. * * @throws IOException Thrown, if the closing of the file stream causes an I/O error. */ @Override public void close() throws IOException { this.wrapBuffer = null; this.readBuffer = null; super.close(); }
public InputSplitOpenThread(FileInputSplit split, long timeout) { super("Transient InputSplit Opener"); setDaemon(true); this.split = split; this.timeout = timeout; }
public LocatableInputSplitAssigner(Collection<LocatableInputSplit> splits) { for(LocatableInputSplit split : splits) { this.unassigned.add(new LocatableInputSplitWithCount(split)); } this.remoteSplitChooser = new LocatableInputSplitChooser(unassigned); }
@Override public String[] readRecord(String[] target, byte[] bytes, int offset, int numBytes) throws IOException { return parseRecord(target, bytes, offset, numBytes) ? target : null; } };
@Override public Value[] readRecord(Value[] target, byte[] bytes, int offset, int numBytes) { return parseRecord(target, bytes, offset, numBytes) ? target : null; } }
private ArrayList<PathMatcher> getExcludeMatchers() { if (excludeMatchers == null) { excludeMatchers = buildPatterns(excludePatterns); } return excludeMatchers; }