/** * Implementation of RecordReader::getCurrentValue(). * @return The value of the current record. (i.e. the target-path.) * @throws IOException * @throws InterruptedException */ @Override public V getCurrentValue() throws IOException, InterruptedException { return chunk.getReader().getCurrentValue(); }
/** * Implementation of RecordReader::getCurrentValue(). * * @return The value of the current record. (i.e. the target-path.) * @throws IOException, on failure. * @throws InterruptedException */ @Override public V getCurrentValue() throws IOException, InterruptedException { return chunk.getReader().getCurrentValue(); }
/** * Implementation of RecordReader::getCurrentValue(). * @return The value of the current record. (i.e. the target-path.) * @throws IOException * @throws InterruptedException */ @Override public V getCurrentValue() throws IOException, InterruptedException { return chunk.getReader().getCurrentValue(); }
/** * Implementation of RecordReader::getCurrentValue(). * * @return The value of the current record. (i.e. the target-path.) * @throws IOException, on failure. * @throws InterruptedException */ @Override public V getCurrentValue() throws IOException, InterruptedException { return chunk.getReader().getCurrentValue(); }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { int count = 0; data.clear(); while (sequenceFileRecordReader.nextKeyValue()) { JSONObject obj = new JSONObject(sequenceFileRecordReader.getCurrentValue().toString()); data.add(obj); count++; if (count == batchSize) { break; } } return (!data.isEmpty()); }
@Override public Tuple getNext() throws IOException { boolean next; try { next = reader.nextKeyValue(); } catch (InterruptedException e) { throw new IOException(e); } if (!next) return null; Object value = reader.getCurrentValue(); if (value == null) { return null; } if(!(value instanceof Tuple)) { return null; } return (Tuple) value; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
/** Read key/value pair in a line. */ public synchronized boolean nextKeyValue() throws IOException, InterruptedException { if (!sequenceFileRecordReader.nextKeyValue()) { return false; } if (key == null) { key = new Text(); } if (value == null) { value = new Text(); } key.set(sequenceFileRecordReader.getCurrentKey().toString()); value.set(sequenceFileRecordReader.getCurrentValue().toString()); return true; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
public synchronized boolean nextKeyValue() throws IOException, InterruptedException { while (super.nextKeyValue()) { key = super.getCurrentKey(); if (filter.accept(key)) { value = super.getCurrentValue(); return true; } } return false; }
@Override public RawRecordContainer getCurrentValue() { // We are going to create a new UID for this event if it contains a digraph in the current UID RawRecordContainer r = super.getCurrentValue(); if (r.getId() != null) { // Create a new UID if the current UID is in the old format String prefix = r.getId().getOptionPrefix(); if ((null != prefix) && (prefix.length() == 2)) { UID newUid = UID.builder().newId(r.getRawData(), r.getTimeForUID()); r.setId(newUid); } } else { // ensure we have an appropriate fatal error r.getErrors().add(RawDataErrorNames.UID_ERROR); } return r; }
private Stream<KV<Text, LongWritable>> extractResultsFromFile(String fileName) { try (SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) { Path path = new Path(fileName); TaskAttemptContext taskContext = HadoopFormats.createTaskAttemptContext(new Configuration(), new JobID("readJob", 0), 0); reader.initialize( new FileSplit(path, 0L, Long.MAX_VALUE, new String[] {"localhost"}), taskContext); List<KV<Text, LongWritable>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add( KV.of( new Text(reader.getCurrentKey().toString()), new LongWritable(reader.getCurrentValue().get()))); } return result.stream(); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public Tuple getNext() throws IOException { boolean next; try { next = reader.nextKeyValue(); } catch (InterruptedException e) { throw new IOException(e); } if (!next) return null; Object value = reader.getCurrentValue(); if (value == null) { return null; } if (!(value instanceof StockPriceWritable)) { return null; } StockPriceWritable w = (StockPriceWritable) value; return TupleFactory.getInstance().newTuple(Arrays.asList( w.getSymbol(), w.getDate(), w.getOpen(), w.getHigh(), w.getLow(), w.getClose(), w.getVolume(), w.getAdjClose() )); }
@Override public Function<String, Stream<Pair<Text, LongWritable>>> extractOutputFunction( String outputDir, Configuration conf) { return part -> ExceptionUtils.unchecked( () -> { try (final SequenceFileRecordReader<Text, LongWritable> reader = new SequenceFileRecordReader<>()) { final Path path = new Path(outputDir + "/" + part); final TaskAttemptContext taskContext = HadoopUtils.createTaskContext(new Configuration(), HadoopUtils.getJobID(), 0); reader.initialize( new FileSplit(path, 0L, Long.MAX_VALUE, new String[] {"localhost"}), taskContext); final List<Pair<Text, LongWritable>> result = new ArrayList<>(); while (reader.nextKeyValue()) { result.add(Pair.of(reader.getCurrentKey(), reader.getCurrentValue())); } return result.stream(); } }); }