/** * Writes a DataSet as text file(s) to the specified location. * * <p>For each element of the DataSet the result of {@link TextFormatter#format(Object)} is written. * * @param filePath The path pointing to the location the text file is written to. * @param writeMode Control the behavior for existing files. Options are NO_OVERWRITE and OVERWRITE. * @param formatter formatter that is applied on every element of the DataSet. * @return The DataSink that writes the DataSet. * * @see TextOutputFormat * @see DataSet#writeAsText(String) Output files and directories */ public DataSink<String> writeAsFormattedText(String filePath, WriteMode writeMode, TextFormatter<T> formatter) { return map(new FormattingMapper<>(clean(formatter))).writeAsText(filePath, writeMode); }
/** * Writes a DataSet as text file(s) to the specified location. * * <p>For each element of the DataSet the result of {@link TextFormatter#format(Object)} is written. * * @param filePath The path pointing to the location the text file is written to. * @param formatter formatter that is applied on every element of the DataSet. * @return The DataSink that writes the DataSet. * * @see TextOutputFormat * @see DataSet#writeAsText(String) Output files and directories */ public DataSink<String> writeAsFormattedText(String filePath, TextFormatter<T> formatter) { return map(new FormattingMapper<>(clean(formatter))).writeAsText(filePath); }
private void createTextSink(PythonOperationInfo info) { DataSet<byte[]> parent = sets.getDataSet(info.parentID); parent.map(new StringDeserializerMap()).setParallelism(info.parallelism) .writeAsText(info.path, info.writeMode).setParallelism(info.parallelism).name("TextSink"); }
@Test public void testSortingParallelism4() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Long> ds = env.generateSequence(0, 1000); // randomize ds.map(new MapFunction<Long, Long>() { Random rand = new Random(1234L); @Override public Long map(Long value) throws Exception { return rand.nextLong(); } }).writeAsText(resultPath) .sortLocalOutput("*", Order.ASCENDING) .setParallelism(4); env.execute(); BufferedReader[] resReaders = getResultReader(resultPath); for (BufferedReader br : resReaders) { long cmp = Long.MIN_VALUE; while (br.ready()) { long cur = Long.parseLong(br.readLine()); assertTrue("Invalid order of sorted output", cmp <= cur); cmp = cur; } br.close(); } }
/** * Writes a DataSet as text file(s) to the specified location. * * <p>For each element of the DataSet the result of {@link TextFormatter#format(Object)} is written. * * @param filePath The path pointing to the location the text file is written to. * @param formatter formatter that is applied on every element of the DataSet. * @return The DataSink that writes the DataSet. * * @see TextOutputFormat * @see DataSet#writeAsText(String) Output files and directories */ public DataSink<String> writeAsFormattedText(String filePath, TextFormatter<T> formatter) { return map(new FormattingMapper<>(clean(formatter))).writeAsText(filePath); }
/** * Writes a DataSet as text file(s) to the specified location. * * <p>For each element of the DataSet the result of {@link TextFormatter#format(Object)} is written. * * @param filePath The path pointing to the location the text file is written to. * @param formatter formatter that is applied on every element of the DataSet. * @return The DataSink that writes the DataSet. * * @see TextOutputFormat * @see DataSet#writeAsText(String) Output files and directories */ public DataSink<String> writeAsFormattedText(String filePath, TextFormatter<T> formatter) { return map(new FormattingMapper<>(clean(formatter))).writeAsText(filePath); }
/** * Writes a DataSet as text file(s) to the specified location. * * <p>For each element of the DataSet the result of {@link TextFormatter#format(Object)} is written. * * @param filePath The path pointing to the location the text file is written to. * @param writeMode Control the behavior for existing files. Options are NO_OVERWRITE and OVERWRITE. * @param formatter formatter that is applied on every element of the DataSet. * @return The DataSink that writes the DataSet. * * @see TextOutputFormat * @see DataSet#writeAsText(String) Output files and directories */ public DataSink<String> writeAsFormattedText(String filePath, WriteMode writeMode, TextFormatter<T> formatter) { return map(new FormattingMapper<>(clean(formatter))).writeAsText(filePath, writeMode); }
/** * Writes a DataSet as text file(s) to the specified location. * * <p>For each element of the DataSet the result of {@link TextFormatter#format(Object)} is written. * * @param filePath The path pointing to the location the text file is written to. * @param writeMode Control the behavior for existing files. Options are NO_OVERWRITE and OVERWRITE. * @param formatter formatter that is applied on every element of the DataSet. * @return The DataSink that writes the DataSet. * * @see TextOutputFormat * @see DataSet#writeAsText(String) Output files and directories */ public DataSink<String> writeAsFormattedText(String filePath, WriteMode writeMode, TextFormatter<T> formatter) { return map(new FormattingMapper<>(clean(formatter))).writeAsText(filePath, writeMode); }