public List<Object> next() throws IOException, ParseException { List<Object> lineTuple = super.next(); if (lineTuple==null) return null; String jsonLine = (String) lineTuple.get(0); if ( jsonLine.trim().isEmpty() ) return next(); try { //1- convert Json to Map<> HashMap<String, Object> jsonMap = new ObjectMapper().readValue(jsonLine, HashMap.class); //2- make StreamlineEvent from map StreamlineEventImpl slEvent = StreamlineEventImpl.builder().putAll(jsonMap).dataSourceId("HdfsSpout").build(); //3- create tuple from StreamlineEvent return Collections.singletonList(slEvent); } catch (JsonProcessingException e) { throw new ParseException("Json parsing error at location : " + getFileOffset().toString(), e); } } }
public TextFileReader(FileSystem fs, Path file, Map conf) throws IOException { this(fs, file, conf, new TextFileReader.Offset(0,0) ); }
public TextFileReader(FileSystem fs, Path file, Map conf, String startOffset) throws IOException { this(fs, file, conf, new TextFileReader.Offset(startOffset) ); }
public Offset getFileOffset() { return offset.clone(); }
@Override public Offset clone() { return new Offset(charOffset, lineNumber); } } //class Offset
public TextFileReader(FileSystem fs, Path file, Map<String, Object> conf, String startOffset) throws IOException { this(fs, file, conf, new TextFileReader.Offset(startOffset)); }
public Offset getFileOffset() { return offset.clone(); }
public TextFileReader(FileSystem fs, Path file, Map<String, Object> conf) throws IOException { this(fs, file, conf, new TextFileReader.Offset(0, 0)); }
@Override public Offset clone() { return new Offset(charOffset, lineNumber); } } //class Offset