/** * Writes a DataStream to the file specified by the path parameter. * * <p>For every field of an element of the DataStream the result of {@link Object#toString()} * is written. This method can only be used on data streams of tuples. * * @param path * the path pointing to the location the text file is written to * * @return the closed DataStream */ @PublicEvolving public DataStreamSink<T> writeAsCsv(String path) { return writeAsCsv(path, null, CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER); }
/** * Writes a DataStream to the file specified by the path parameter. * * <p>For every field of an element of the DataStream the result of {@link Object#toString()} * is written. This method can only be used on data streams of tuples. * * @param path * the path pointing to the location the text file is written to * @param writeMode * Controls the behavior for existing files. Options are * NO_OVERWRITE and OVERWRITE. * * @return the closed DataStream */ @PublicEvolving public DataStreamSink<T> writeAsCsv(String path, WriteMode writeMode) { return writeAsCsv(path, writeMode, CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER); }
@Test public void testProgram() throws Exception { String resultPath = getTempDirPath("result"); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<String> text = env.fromElements(WordCountData.TEXT); DataStream<Tuple2<String, Integer>> counts = text .flatMap(new Tokenizer()) .keyBy(0).sum(1); counts.writeAsCsv(resultPath); env.execute("WriteAsCsvTest"); //Strip the parentheses from the expected text like output compareResultsByLinesInMemory(WordCountData.STREAMING_COUNTS_AS_TUPLES .replaceAll("[\\\\(\\\\)]", ""), resultPath); }
/** * Writes a DataStream to the file specified by the path parameter. * * <p>For every field of an element of the DataStream the result of {@link Object#toString()} * is written. This method can only be used on data streams of tuples. * * @param path * the path pointing to the location the text file is written to * * @return the closed DataStream */ @PublicEvolving public DataStreamSink<T> writeAsCsv(String path) { return writeAsCsv(path, null, CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER); }
/** * Writes a DataStream to the file specified by the path parameter. * * <p>For every field of an element of the DataStream the result of {@link Object#toString()} * is written. This method can only be used on data streams of tuples. * * @param path * the path pointing to the location the text file is written to * * @return the closed DataStream */ @PublicEvolving public DataStreamSink<T> writeAsCsv(String path) { return writeAsCsv(path, null, CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER); }
/** * Writes a DataStream to the file specified by the path parameter. * * <p>For every field of an element of the DataStream the result of {@link Object#toString()} * is written. This method can only be used on data streams of tuples. * * @param path * the path pointing to the location the text file is written to * * @return the closed DataStream */ @PublicEvolving public DataStreamSink<T> writeAsCsv(String path) { return writeAsCsv(path, null, CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER); }
/** * Writes a DataStream to the file specified by the path parameter. * * <p>For every field of an element of the DataStream the result of {@link Object#toString()} * is written. This method can only be used on data streams of tuples. * * @param path * the path pointing to the location the text file is written to * @param writeMode * Controls the behavior for existing files. Options are * NO_OVERWRITE and OVERWRITE. * * @return the closed DataStream */ @PublicEvolving public DataStreamSink<T> writeAsCsv(String path, WriteMode writeMode) { return writeAsCsv(path, writeMode, CsvOutputFormat.DEFAULT_LINE_DELIMITER, CsvOutputFormat.DEFAULT_FIELD_DELIMITER); }
public void testWithDiscardFilter() throws Exception { /* * Test filterVertices() with a filter that constantly returns false */ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); GraphStream<Long, NullValue, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); graph = graph.filterVertices(new DiscardFilter()); graph.getEdges().writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = ""; }
public void testWithDiscardFilter() throws Exception { /* * Test filterEdges() with a filter that constantly returns false */ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); GraphStream<Long, NullValue, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); graph = graph.filterEdges(new DiscardFilter()); graph.getEdges().writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = ""; }
public void testWithSimpleFilter() throws Exception { /* * Test filterVertices() with a simple filter */ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); GraphStream<Long, NullValue, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); graph = graph.filterVertices(new LowVertexKeyFilter()); graph.getEdges().writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "2,3,23\n" + "3,4,34\n" + "3,5,35\n" + "4,5,45\n"; }
public void testWithSimpleFilter() throws Exception { /* * Test filterEdges() with a simple filter */ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); GraphStream<Long, NullValue, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); graph = graph.filterEdges(new LowEdgeValueFilter()); graph.getEdges().writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "2,3,23\n" + "3,4,34\n" + "3,5,35\n" + "4,5,45\n" + "5,1,51\n"; }
public void testReduceOnNeighborsDefault() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SimpleEdgeStream<Long, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); DataStream<Tuple2<Long, Long>> sum = graph.slice(Time.of(1, TimeUnit.SECONDS)) .reduceOnEdges(new SumEdgeValuesReduce()); sum.writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "1,25\n" + "2,23\n" + "3,69\n" + "4,45\n" + "5,51\n"; }
public void testApplyOnNeighborsDefault() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SimpleEdgeStream<Long, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); DataStream<Tuple2<Long, String>> sum = graph.slice(Time.of(1, TimeUnit.SECONDS)) .applyOnNeighbors(new SumEdgeValuesApply()); sum.writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "1,small\n" + "2,small\n" + "3,big\n" + "4,small\n" + "5,big\n"; }
public void testApplyOnNeighborsAll() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SimpleEdgeStream<Long, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); DataStream<Tuple2<Long, String>> sum = graph.slice(Time.of(1, TimeUnit.SECONDS), EdgeDirection.ALL) .applyOnNeighbors(new SumEdgeValuesApply()); sum.writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "1,big\n" + "2,small\n" + "3,big\n" + "4,big\n" + "5,big\n"; }
public void testReduceOnNeighborsIn() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SimpleEdgeStream<Long, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); DataStream<Tuple2<Long, Long>> sum = graph.slice(Time.of(1, TimeUnit.SECONDS), EdgeDirection.IN) .reduceOnEdges(new SumEdgeValuesReduce()); sum.writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "1,51\n" + "2,12\n" + "3,36\n" + "4,34\n" + "5,80\n"; }
public void testReduceOnNeighborsAll() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SimpleEdgeStream<Long, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); DataStream<Tuple2<Long, Long>> sum = graph.slice(Time.of(1, TimeUnit.SECONDS), EdgeDirection.ALL) .reduceOnEdges(new SumEdgeValuesReduce()); sum.writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "1,76\n" + "2,35\n" + "3,105\n" + "4,79\n" + "5,131\n"; }
public void testApplyOnNeighborsIn() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SimpleEdgeStream<Long, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); DataStream<Tuple2<Long, String>> sum = graph.slice(Time.of(1, TimeUnit.SECONDS), EdgeDirection.IN) .applyOnNeighbors(new SumEdgeValuesApply()); sum.writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "1,big\n" + "2,small\n" + "3,small\n" + "4,small\n" + "5,big\n"; }
public void testFoldNeighborsAll() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SimpleEdgeStream<Long, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); DataStream<Tuple2<Long, Long>> sum = graph.slice(Time.of(1, TimeUnit.SECONDS), EdgeDirection.ALL) .foldNeighbors(new Tuple2<Long, Long>(0l, 0l), new SumEdgeValues()); sum.writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "1,76\n" + "2,35\n" + "3,105\n" + "4,79\n" + "5,131\n"; }
public void testFoldNeighborsDefault() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SimpleEdgeStream<Long, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); DataStream<Tuple2<Long, Long>> sum = graph.slice(Time.of(1, TimeUnit.SECONDS)) .foldNeighbors(new Tuple2<Long, Long>(0l, 0l), new SumEdgeValues()); sum.writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "1,25\n" + "2,23\n" + "3,69\n" + "4,45\n" + "5,51\n"; }
public void testFoldNeighborsIn() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SimpleEdgeStream<Long, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env); DataStream<Tuple2<Long, Long>> sum = graph.slice(Time.of(1, TimeUnit.SECONDS), EdgeDirection.IN) .foldNeighbors(new Tuple2<Long, Long>(0l, 0l), new SumEdgeValues()); sum.writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE); env.execute(); expectedResult = "1,51\n" + "2,12\n" + "3,36\n" + "4,34\n" + "5,80\n"; }