/** * Writes the dataStream into an output, described by an OutputFormat. * * <p>The output is not participating in Flink's checkpointing! * * <p>For writing to a file system periodically, the use of the "flink-connector-filesystem" * is recommended. * * @param format The output format * @return The closed DataStream */ @PublicEvolving public DataStreamSink<T> writeUsingOutputFormat(OutputFormat<T> format) { return addSink(new OutputFormatSinkFunction<>(format)); }
@Override public void invoke(IN record) throws Exception { try { format.writeRecord(record); } catch (Exception ex) { cleanup(); throw ex; } }
@Override public void open(Configuration parameters) throws Exception { RuntimeContext context = getRuntimeContext(); format.configure(parameters); int indexInSubtaskGroup = context.getIndexOfThisSubtask(); int currentNumberOfSubtasks = context.getNumberOfParallelSubtasks(); format.open(indexInSubtaskGroup, currentNumberOfSubtasks); }
@Test public void setRuntimeContext() throws Exception { RuntimeContext mockRuntimeContext = Mockito.mock(RuntimeContext.class); // Make sure setRuntimeContext of the rich output format is called RichOutputFormat<?> mockRichOutputFormat = Mockito.mock(RichOutputFormat.class); new OutputFormatSinkFunction<>(mockRichOutputFormat).setRuntimeContext(mockRuntimeContext); Mockito.verify(mockRichOutputFormat, Mockito.times(1)).setRuntimeContext(Mockito.eq(mockRuntimeContext)); // Make sure setRuntimeContext work well when output format is not RichOutputFormat OutputFormat<?> mockOutputFormat = Mockito.mock(OutputFormat.class); new OutputFormatSinkFunction<>(mockOutputFormat).setRuntimeContext(mockRuntimeContext); } }
/** * Writes the dataStream into an output, described by an OutputFormat. * * <p>The output is not participating in Flink's checkpointing! * * <p>For writing to a file system periodically, the use of the "flink-connector-filesystem" * is recommended. * * @param format The output format * @return The closed DataStream */ @PublicEvolving public DataStreamSink<T> writeUsingOutputFormat(OutputFormat<T> format) { return addSink(new OutputFormatSinkFunction<>(format)); }
@Override public void close() throws IOException { try { format.close(); } catch (Exception ex) { cleanup(); throw ex; } }
@Override public void open(Configuration parameters) throws Exception { RuntimeContext context = getRuntimeContext(); format.configure(parameters); int indexInSubtaskGroup = context.getIndexOfThisSubtask(); int currentNumberOfSubtasks = context.getNumberOfParallelSubtasks(); format.open(indexInSubtaskGroup, currentNumberOfSubtasks); }
/** * Writes the dataStream into an output, described by an OutputFormat. * * <p>The output is not participating in Flink's checkpointing! * * <p>For writing to a file system periodically, the use of the "flink-connector-filesystem" * is recommended. * * @param format The output format * @return The closed DataStream */ @PublicEvolving public DataStreamSink<T> writeUsingOutputFormat(OutputFormat<T> format) { return addSink(new OutputFormatSinkFunction<>(format)); }
@Override public void close() throws IOException { try { format.close(); } catch (Exception ex) { cleanup(); throw ex; } }
@Override public void open(Configuration parameters) throws Exception { RuntimeContext context = getRuntimeContext(); format.configure(parameters); int indexInSubtaskGroup = context.getIndexOfThisSubtask(); int currentNumberOfSubtasks = context.getNumberOfParallelSubtasks(); format.open(indexInSubtaskGroup, currentNumberOfSubtasks); }
/** * Writes the dataStream into an output, described by an OutputFormat. * * <p>The output is not participating in Flink's checkpointing! * * <p>For writing to a file system periodically, the use of the "flink-connector-filesystem" * is recommended. * * @param format The output format * @return The closed DataStream */ @PublicEvolving public DataStreamSink<T> writeUsingOutputFormat(OutputFormat<T> format) { return addSink(new OutputFormatSinkFunction<>(format)); }
@Override public void invoke(IN record) throws Exception { try { format.writeRecord(record); } catch (Exception ex) { cleanup(); throw ex; } }
@Override public void open(Configuration parameters) throws Exception { RuntimeContext context = getRuntimeContext(); format.configure(parameters); int indexInSubtaskGroup = context.getIndexOfThisSubtask(); int currentNumberOfSubtasks = context.getNumberOfParallelSubtasks(); format.open(indexInSubtaskGroup, currentNumberOfSubtasks); }
@Override public DataStreamSink<?> writeData(DataStream<Row> dataSet) { StreamOutputFormatBuilder builder = new StreamOutputFormatBuilder(); builder.setPrint(print); OutputFormatSinkFunction formatSinkFunction = new OutputFormatSinkFunction(builder.finish()); DataStreamSink<?> dataStreamSink = dataSet.addSink(formatSinkFunction); dataStreamSink.name("streamwriter"); return dataStreamSink; } }
@Override public void close() throws IOException { try { format.close(); } catch (Exception ex) { cleanup(); throw ex; } }
@Override public DataStreamSink<?> writeData(DataStream<Row> dataSet) { JdbcOutputFormatBuilder builder = new JdbcOutputFormatBuilder(); builder.setDrivername(databaseInterface.getDriverClass()); builder.setDBUrl(dbUrl); builder.setUsername(username); builder.setPassword(password); builder.setBatchInterval(getBatchSize()); builder.setMonitorUrls(monitorUrls); builder.setPreSql(preSql); builder.setPostSql(postSql); builder.setErrors(errors); builder.setErrorRatio(errorRatio); builder.setDirtyPath(dirtyPath); builder.setDirtyHadoopConfig(dirtyHadoopConfig); builder.setSrcCols(srcCols); builder.setDatabaseInterface(databaseInterface); builder.setMode(mode); builder.setTable(table); builder.setColumn(column); builder.setFullColumn(fullColumn); builder.setUpdateKey(updateKey); builder.setTypeConverter(typeConverter); OutputFormatSinkFunction sinkFunction = new OutputFormatSinkFunction(builder.finish()); DataStreamSink<?> dataStreamSink = dataSet.addSink(sinkFunction); String sinkName = (databaseInterface.getDatabaseType() + "writer").toLowerCase(); dataStreamSink.name(sinkName); return dataStreamSink; }
@Override public void invoke(IN record) throws Exception { try { format.writeRecord(record); } catch (Exception ex) { cleanup(); throw ex; } }
@Override public void close() throws IOException { try { format.close(); } catch (Exception ex) { cleanup(); throw ex; } }
@Override public void invoke(IN record) throws Exception { try { format.writeRecord(record); } catch (Exception ex) { cleanup(); throw ex; } }