Refine search
@Override public void sourcePrepare(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) throws IOException { super.sourcePrepare(flowProcess, sourceCall); Object[] context = new Object[SRC_CTX_SIZE]; context[SRC_CTX_KEY] = sourceCall.getInput().createKey(); context[SRC_CTX_VALUE] = sourceCall.getInput().createValue(); // as the tuple _might_ vary (some objects might be missing), we use a map rather then a collection Settings settings = loadSettings(flowProcess.getConfigCopy(), true); context[SRC_CTX_ALIASES] = CascadingUtils.alias(settings); context[SRC_CTX_OUTPUT_JSON] = settings.getOutputAsJson(); sourceCall.setContext(context); }
@SuppressWarnings("unchecked") @Override public boolean source(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) throws IOException { Object[] context = sourceCall.getContext(); if (!sourceCall.getInput().next(context[SRC_CTX_KEY], context[1])) { return false; TupleEntry entry = sourceCall.getIncomingEntry(); if (entry.getFields().isDefined()) { for (Comparable<?> field : entry.getFields()) { List<Object> elements = Tuple.elements(entry.getTuple()); elements.clear(); elements.addAll(data.values());
@Override public void sourceCleanup(FlowProcess<Properties> flowProcess, SourceCall<Object[], ScrollQuery> sourceCall) throws IOException { // in case of a source there's no local client so do all reporting here report(sourceCall.getInput().stats(), flowProcess); report(sourceCall.getInput().repository().stats(), flowProcess); sourceCall.getInput().close(); sourceCall.setContext(null); // used for consistency cleanupClient(flowProcess); }
@SuppressWarnings("unchecked") @Override public boolean source(FlowProcess<JobConf> fp, SourceCall<Object[], RecordReader> sc) throws IOException { Container<Tuple> value = (Container<Tuple>) sc.getInput().createValue(); boolean hasNext = sc.getInput().next(null, value); if (!hasNext) { return false; } // Skip nulls if (value == null) { return true; } sc.getIncomingEntry().setTuple(value.get()); return true; }
@Override public boolean source(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader<TupleWrapper, NullWritable>> sourceCall) throws IOException { TupleWrapper key = (TupleWrapper) sourceCall.getContext()[ 0 ]; NullWritable value = (NullWritable) sourceCall.getContext()[ 1 ]; boolean result = sourceCall.getInput().next( key, value ); if( !result ) return false; sourceCall.getIncomingEntry().setTuple(key.tuple); return true; }
@Override public boolean source(FlowProcess<JobConf> jobConfFlowProcess, SourceCall<Object[], RecordReader> sourceCall) throws IOException { Tuple tuple = sourceCall.getIncomingEntry().getTuple(); IntWritable partition = (IntWritable) sourceCall.getInput().createKey(); boolean result = sourceCall.getInput().next(partition, null); if(!result){ return false; } tuple.set(0, null); tuple.set(1, null); tuple.set(2, partition); tuple.set(3, null); return true; }
private boolean sourceReadInput( SourceCall<Object[], RecordReader> sourceCall ) throws IOException { Object[] context = sourceCall.getContext(); return sourceCall.getInput().next( context[ 0 ], context[ 1 ] ); }
@Override public boolean source( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) throws IOException { Tuple key = (Tuple) sourceCall.getContext()[ 0 ]; Tuple value = (Tuple) sourceCall.getContext()[ 1 ]; boolean result = sourceCall.getInput().next( key, value ); if( !result ) return false; TupleEntry entry = sourceCall.getIncomingEntry(); if( entry.hasTypes() ) entry.setCanonicalTuple( value ); else entry.setTuple( value ); return true; }
@Override public boolean source( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) throws IOException { Object key = sourceCall.getContext()[ 0 ]; Object value = sourceCall.getContext()[ 1 ]; boolean result = sourceCall.getInput().next( key, value ); if( !result ) return false; int count = 0; TupleEntry entry = sourceCall.getIncomingEntry(); if( keyType != null ) entry.setObject( count++, key ); if( valueType != null ) entry.setObject( count, value ); return true; }
@SuppressWarnings("unchecked") @Override public boolean source(FlowProcess<Properties> flowProcess, SourceCall<Object[], ScrollQuery> sourceCall) throws IOException { ScrollQuery query = sourceCall.getInput(); boolean isJSON = (Boolean) sourceCall.getContext()[SRC_CTX_OUTPUT_JSON]; TupleEntry entry = sourceCall.getIncomingEntry(); FieldAlias alias = (FieldAlias) sourceCall.getContext()[SRC_CTX_ALIASES]; if (entry.getFields().isDefined()) { for (Comparable<?> field : entry.getFields()) { Object result = data; entry.setObject(field, result);
@Override public void sourcePrepare( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { sourceCall.setContext( createInput( sourceCall.getInput() ) ); sourceCall.getIncomingEntry().setTuple( TupleViews.createObjectArray() ); }
/** {@inheritDoc} */ @Override public boolean source(FlowProcess<? extends Properties> flowProcess, SourceCall<Void, Iterator<Tuple>> sourceCall) throws IOException { if (sourceCall.getInput().hasNext()) { sourceCall.getIncomingEntry().setTuple(sourceCall.getInput().next()); return true; } return false; }
@Override public boolean source( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { String line = sourceCall.getContext().readLine(); if( line == null ) return false; if( skipHeader && sourceCall.getContext().getLineNumber() == 1 ) // todo: optimize this away line = sourceCall.getContext().readLine(); if( line == null ) return false; Object[] split = delimitedParser.parseLine( line ); // assumption it is better to re-use than to construct new Tuple tuple = sourceCall.getIncomingEntry().getTuple(); TupleViews.reset( tuple, split ); return true; }
@Override public void sourcePrepare( FlowProcess<? extends Configuration> flowProcess, SourceCall<Object[], RecordReader> sourceCall ) { super.sourcePrepare( flowProcess, sourceCall ); sourceCall.getIncomingEntry().setTuple( TupleViews.createObjectArray() ); }
@Override public boolean source( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { // first line is 0, this matches offset being zero, so when throwing out the first line for comments int lineNumber = sourceCall.getContext().getLineNumber(); String line = sourceCall.getContext().readLine(); if( line == null ) return false; TupleEntry incomingEntry = sourceCall.getIncomingEntry(); if( getSourceFields().size() == 1 ) { incomingEntry.setObject( 0, line ); } else { incomingEntry.setInteger( 0, lineNumber ); incomingEntry.setString( 1, line ); } return true; }
@Override public boolean source( FlowProcess<? extends Properties> flowProcess, SourceCall<LineNumberReader, InputStream> sourceCall ) throws IOException { String line = sourceCall.getContext().readLine(); if( line == null ) return false; TupleEntry incomingEntry = sourceCall.getIncomingEntry(); JsonNode jsonNode = null; if( !line.isEmpty() ) jsonNode = mapper.readTree( line ); incomingEntry.setObject( 0, jsonNode ); return true; }
protected void sourceHandleInput( SourceCall<Object[], RecordReader> sourceCall ) throws IOException { TupleEntry result = sourceCall.getIncomingEntry(); int index = 0; Object[] context = sourceCall.getContext(); // coerce into canonical forms if( getSourceFields().size() == 2 ) result.setLong( index++, ( (LongWritable) context[ 0 ] ).get() ); result.setString( index, makeEncodedString( context ) ); }
@Override public void sourceCleanup(FlowProcess<JobConf> flowProcess, SourceCall<Object[], RecordReader> sourceCall) throws IOException { super.sourceCleanup(flowProcess, sourceCall); sourceCall.setContext(null); }
@SuppressWarnings("unchecked") @Override public boolean source(FlowProcess<? extends JobConf> fp, SourceCall<Object[], RecordReader> sc) throws IOException { Container<Tuple> value = (Container<Tuple>) sc.getInput().createValue(); boolean hasNext = sc.getInput().next(null, value); if (!hasNext) { return false; } // Skip nulls if (value == null) { return true; } sc.getIncomingEntry().setTuple(value.get()); return true; }
@Override public boolean source(FlowProcess<JobConf> prcs, SourceCall<Object[], RecordReader> sourceCall) throws IOException { Object key = sourceCall.getContext()[0]; Object value = sourceCall.getContext()[1]; boolean result = sourceCall.getInput().next(key, value); if (!result) { return false; } sourceCall.getIncomingEntry().setTuple(new Tuple(key, value)); return true; }