@Override public void open(FileInputSplit split) throws IOException { super.open(split); dataFileReader = initReader(split); dataFileReader.sync(split.getStart()); lastSync = dataFileReader.previousSync(); }
byte prev = 0; for (FileInputSplit inputSplit : inputSplits) { inputFormat.open(inputSplit); while (!inputFormat.reachedEnd()) { if ((bytes = inputFormat.nextRecord(bytes)) != null) {
@Override public void open(FileInputSplit split) throws IOException { super.open(split); this.blockInfo = this.createAndReadBlockInfo(); // We set the size of the BlockBasedInput to splitLength as each split contains one block. // After reading the block info, we seek in the file to the correct position. this.readRecords = 0; this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockInfo.getFirstRecordStart(), this.splitLength); this.dataInputStream = new DataInputViewStreamWrapper(blockBasedInput); }
/** * Opens the given input split. This method opens the input stream to the specified file, allocates read buffers * and positions the stream at the correct position, making sure that any partial record at the beginning is skipped. * * @param split The input split to open. * * @see org.apache.flink.api.common.io.FileInputFormat#open(org.apache.flink.core.fs.FileInputSplit) */ @Override public void open(FileInputSplit split) throws IOException { super.open(split); initBuffers(); this.offset = splitStart; if (this.splitStart != 0) { this.stream.seek(offset); readLine(); // if the first partial record already pushes the stream over // the limit of our split, then no record starts within this split if (this.overLimit) { this.end = true; } } else { fillBuffer(0); } }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); LOG.debug("Initialize Bitcoin reader"); // temporary measure to set buffer size to 1, otherwise we cannot guarantee that checkpointing works bbr = new BitcoinBlockReader(this.stream,this.maxSizeBitcoinBlock,1,this.specificMagicArray,this.useDirectBuffer,this.readAuxPOW); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); LOG.debug("Initialize Ethereum reader"); // temporary measure to set buffer size to 1, otherwise we cannot guarantee that checkpointing works ebr = new EthereumBlockReader(this.stream,this.maxSizeEthereumBlock,1,this.useDirectBuffer); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); dataFileReader = initReader(split); dataFileReader.sync(split.getStart()); lastSync = dataFileReader.previousSync(); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); dataFileReader = initReader(split); dataFileReader.sync(split.getStart()); lastSync = dataFileReader.previousSync(); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); dataFileReader = initReader(split); dataFileReader.sync(split.getStart()); lastSync = dataFileReader.previousSync(); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); dataFileReader = initReader(split); dataFileReader.sync(split.getStart()); lastSync = dataFileReader.previousSync(); }
/** * Opens the given input split. This method opens the input stream to the specified file, allocates read buffers * and positions the stream at the correct position, making sure that any partial record at the beginning is skipped. * * @param split The input split to open. * * @see org.apache.flink.api.common.io.FileInputFormat#open(org.apache.flink.core.fs.FileInputSplit) */ @Override public void open(FileInputSplit split) throws IOException { super.open(split); initBuffers(); this.offset = splitStart; if (this.splitStart != 0) { this.stream.seek(offset); readLine(); // if the first partial record already pushes the stream over // the limit of our split, then no record starts within this split if (this.overLimit) { this.end = true; } } else { fillBuffer(0); } }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); this.blockInfo = this.createAndReadBlockInfo(); // We set the size of the BlockBasedInput to splitLength as each split contains one block. // After reading the block info, we seek in the file to the correct position. this.readRecords = 0; this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockInfo.getFirstRecordStart(), this.splitLength); this.dataInputStream = new DataInputViewStreamWrapper(blockBasedInput); }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); this.blockInfo = this.createAndReadBlockInfo(); // We set the size of the BlockBasedInput to splitLength as each split contains one block. // After reading the block info, we seek in the file to the correct position. this.readRecords = 0; this.stream.seek(this.splitStart + this.blockInfo.getFirstRecordStart()); this.blockBasedInput = new BlockBasedInput(this.stream, (int) blockInfo.getFirstRecordStart(), this.splitLength); this.dataInputStream = new DataInputViewStreamWrapper(blockBasedInput); }
/** * Opens the given input split. This method opens the input stream to the specified file, allocates read buffers * and positions the stream at the correct position, making sure that any partial record at the beginning is skipped. * * @param split The input split to open. * * @see org.apache.flink.api.common.io.FileInputFormat#open(org.apache.flink.core.fs.FileInputSplit) */ @Override public void open(FileInputSplit split) throws IOException { super.open(split); initBuffers(); this.offset = splitStart; if (this.splitStart != 0) { this.stream.seek(offset); readLine(); // if the first partial record already pushes the stream over // the limit of our split, then no record starts within this split if (this.overLimit) { this.end = true; } } else { fillBuffer(0); } }
@Override public void open(FileInputSplit split) throws IOException { super.open(split); this.reachedEnd=false; this.hocr.setFileName(split.getPath().getName());