@Override public RecordReader createReader(InputSplit split) throws IOException, InterruptedException { LineRecordReader ret = new LineRecordReader(); ret.initialize(split); return ret; }
@Override public void close() throws IOException { super.close(); }
@Override public Configuration getConf() { return super.getConf(); }
@Override public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException { initialize(split); }
@Override public Collection<Writable> next() { Text t = (Text) super.next().iterator().next(); String val = new String(t.getBytes()); String[] split = val.split(","); List<Writable> ret = new ArrayList<>(); for(String s : split) ret.add(new Text(s)); return ret; } }
@Override public Collection<Writable> next() { List<Writable> ret = new ArrayList<>(); if(iter.hasNext()) { String record = iter.next(); invokeListeners(record); ret.add(new Text(record)); return ret; } else { if ( !(inputSplit instanceof StringSplit) && currIndex < locations.length-1 ) { currIndex++; try { close(); iter = IOUtils.lineIterator(new InputStreamReader(locations[currIndex].toURL().openStream())); } catch (IOException e) { e.printStackTrace(); } if(iter.hasNext()) { String record = iter.next(); invokeListeners(record); ret.add(new Text(record)); return ret; } } throw new NoSuchElementException("No more elements found!"); } }
@Override public void reset() { super.reset(); skippedLines = false; } }
@Override public void setConf(Configuration conf) { super.setConf(conf); }
@Override public boolean hasNext() { return super.hasNext(); }
@Override public Collection<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException { invokeListeners(uri); //Here: we are reading a single line from the DataInputStream BufferedReader br = new BufferedReader(new InputStreamReader(dataInputStream)); String line = br.readLine(); return Collections.singletonList((Writable)new Text(line)); } }
@Override public Collection<Writable> record(URI uri, DataInputStream dataInputStream) throws IOException { Writable w = ((List<Writable>)super.record(uri,dataInputStream)).get(0); return getRecord(w.toString()); }
@Override public void initialize(InputSplit split) throws IOException, InterruptedException { super.initialize(split); }
@Override public Collection<Writable> next() { if(numLinesSkipped < skipNumLines) { for(int i = numLinesSkipped; i < skipNumLines; i++, numLinesSkipped++) { if(!hasNext()) { return new ArrayList<>(); } super.next(); } } Text t = (Text) super.next().iterator().next(); String val = t.toString(); return getRecord(val); }
@Override public void reset(){ super.reset(); numLinesSkipped = 0; }
@Override public void setConf(Configuration conf) { super.setConf(conf); if(conf.get(NUM_ATTRIBUTES) != null) numAttributes = conf.getInt(NUM_ATTRIBUTES,-1); }
@Override public RecordReader createReader(InputSplit split) throws IOException, InterruptedException { LineRecordReader ret = new LineRecordReader(); ret.initialize(split); return ret; }
@Override public void reset() { if(inputSplit == null) throw new UnsupportedOperationException("Cannot reset without first initializing"); try{ initialize(inputSplit); }catch(Exception e){ throw new RuntimeException("Error during LineRecordReader reset",e); } }
@Override public Collection<Writable> next() { if(!skippedLines && skipNumLines > 0) { for(int i = 0; i < skipNumLines; i++) { if(!hasNext()) { return new ArrayList<>(); } super.next(); } skippedLines = true; } Text t = (Text) super.next().iterator().next(); String val = t.toString(); String[] split = val.split(delimiter, -1); List<Writable> ret = new ArrayList<>(); for(String s : split) ret.add(new Text(s)); return ret; }
@Override public boolean hasNext() { if ( iter != null && iter.hasNext() ) { return true; } else { if (locations != null && !(inputSplit instanceof StringSplit) && currIndex < locations.length-1 ) { currIndex++; try { close(); iter = IOUtils.lineIterator(new InputStreamReader(locations[currIndex].toURL().openStream())); } catch (IOException e) { e.printStackTrace(); } return iter.hasNext(); } return false; } }
@Override public void initialize(Configuration conf, InputSplit split) throws IOException, InterruptedException { super.initialize(conf, split); this.skipNumLines = conf.getInt(SKIP_NUM_LINES,this.skipNumLines); }