@Override public void open(Configuration parameters) throws Exception { super.open(parameters); format.configure(parameters); if (LOG.isDebugEnabled()) { LOG.debug("Opened {} (taskIdx= {}) for path: {}", getClass().getSimpleName(), getRuntimeContext().getIndexOfThisSubtask(), path); } }
@Override public void configure(Configuration parameters) { super.configure(parameters); // the if is to prevent the configure() method from // overwriting the value set by the setter if (this.blockSize == NATIVE_BLOCK_SIZE) { long blockSize = parameters.getLong(BLOCK_SIZE_PARAMETER_KEY, NATIVE_BLOCK_SIZE); setBlockSize(blockSize); } }
super.configure(parameters);
inputFormat.setFilePath(tempFile.toURI().toString()); inputFormat.configure(config); inputFormat.openInputFormat();
@Override public void open() throws Exception { super.open(); checkState(this.reader == null, "The reader is already initialized."); checkState(this.serializer != null, "The serializer has not been set. " + "Probably the setOutputType() was not called. Please report it."); this.format.setRuntimeContext(getRuntimeContext()); this.format.configure(new Configuration()); this.checkpointLock = getContainingTask().getCheckpointLock(); // set the reader context based on the time characteristic final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic(); final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval(); this.readerContext = StreamSourceContexts.getSourceContext( timeCharacteristic, getProcessingTimeService(), checkpointLock, getContainingTask().getStreamStatusMaintainer(), output, watermarkInterval, -1); // and initialize the split reading thread this.reader = new SplitReader<>(format, serializer, readerContext, checkpointLock, restoredReaderState); this.restoredReaderState = null; this.reader.start(); }
@Test public void testCreateInputSplitSingleFile() throws IOException { String tempFile = TestFileUtils.createTempFile("Hello World"); FileInputFormat fif = new DummyFileInputFormat(); fif.setFilePath(tempFile); fif.configure(new Configuration()); FileInputSplit[] splits = fif.createInputSplits(2); Assert.assertEquals(2, splits.length); Assert.assertEquals(tempFile, splits[0].getPath().toString()); Assert.assertEquals(tempFile, splits[1].getPath().toString()); }
fif.setFilePaths(tempFile1, tempFile2, tempFile3); fif.configure(new Configuration()); FileInputSplit[] splits = fif.createInputSplits(3);
@Override public void open(Configuration parameters) throws Exception { super.open(parameters); format.configure(parameters); if (LOG.isDebugEnabled()) { LOG.debug("Opened {} (taskIdx= {}) for path: {}", getClass().getSimpleName(), getRuntimeContext().getIndexOfThisSubtask(), path); } }
@Override public void open(Configuration parameters) throws Exception { super.open(parameters); format.configure(parameters); if (LOG.isDebugEnabled()) { LOG.debug("Opened {} (taskIdx= {}) for path: {}", getClass().getSimpleName(), getRuntimeContext().getIndexOfThisSubtask(), path); } }
@Override public void open(Configuration parameters) throws Exception { super.open(parameters); format.configure(parameters); if (LOG.isDebugEnabled()) { LOG.debug("Opened {} (taskIdx= {}) for path: {}", getClass().getSimpleName(), getRuntimeContext().getIndexOfThisSubtask(), path); } }
@Override public void configure(Configuration parameters) { super.configure(parameters); // the if is to prevent the configure() method from // overwriting the value set by the setter if (this.blockSize == NATIVE_BLOCK_SIZE) { long blockSize = parameters.getLong(BLOCK_SIZE_PARAMETER_KEY, NATIVE_BLOCK_SIZE); setBlockSize(blockSize); } }
@Override public void configure(Configuration parameters) { super.configure(parameters); // the if is to prevent the configure() method from // overwriting the value set by the setter if (this.blockSize == NATIVE_BLOCK_SIZE) { long blockSize = parameters.getLong(BLOCK_SIZE_PARAMETER_KEY, NATIVE_BLOCK_SIZE); setBlockSize(blockSize); } }
super.configure(parameters);
super.configure(parameters);
@Override public void open() throws Exception { super.open(); checkState(this.reader == null, "The reader is already initialized."); checkState(this.serializer != null, "The serializer has not been set. " + "Probably the setOutputType() was not called. Please report it."); this.format.setRuntimeContext(getRuntimeContext()); this.format.configure(new Configuration()); this.checkpointLock = getContainingTask().getCheckpointLock(); // set the reader context based on the time characteristic final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic(); final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval(); this.readerContext = StreamSourceContexts.getSourceContext( timeCharacteristic, getProcessingTimeService(), checkpointLock, getContainingTask().getStreamStatusMaintainer(), output, watermarkInterval, -1); // and initialize the split reading thread this.reader = new SplitReader<>(format, serializer, readerContext, checkpointLock, restoredReaderState); this.restoredReaderState = null; this.reader.start(); }
@Override public void open() throws Exception { super.open(); checkState(this.reader == null, "The reader is already initialized."); checkState(this.serializer != null, "The serializer has not been set. " + "Probably the setOutputType() was not called. Please report it."); this.format.setRuntimeContext(getRuntimeContext()); this.format.configure(new Configuration()); this.checkpointLock = getContainingTask().getCheckpointLock(); // set the reader context based on the time characteristic final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic(); final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval(); this.readerContext = StreamSourceContexts.getSourceContext( timeCharacteristic, getProcessingTimeService(), checkpointLock, getContainingTask().getStreamStatusMaintainer(), output, watermarkInterval, -1); // and initialize the split reading thread this.reader = new SplitReader<>(format, serializer, readerContext, checkpointLock, restoredReaderState); this.restoredReaderState = null; this.reader.start(); }
@Override public void open() throws Exception { super.open(); checkState(this.reader == null, "The reader is already initialized."); checkState(this.serializer != null, "The serializer has not been set. " + "Probably the setOutputType() was not called. Please report it."); this.format.setRuntimeContext(getRuntimeContext()); this.format.configure(new Configuration()); this.checkpointLock = getContainingTask().getCheckpointLock(); // set the reader context based on the time characteristic final TimeCharacteristic timeCharacteristic = getOperatorConfig().getTimeCharacteristic(); final long watermarkInterval = getRuntimeContext().getExecutionConfig().getAutoWatermarkInterval(); this.readerContext = StreamSourceContexts.getSourceContext( timeCharacteristic, getProcessingTimeService(), checkpointLock, getContainingTask().getStreamStatusMaintainer(), output, watermarkInterval, -1); // and initialize the split reading thread this.reader = new SplitReader<>(format, serializer, readerContext, checkpointLock, restoredReaderState); this.restoredReaderState = null; this.reader.start(); }