Refine search
static Tuple coerceToString(SinkCall<?, ?> sinkCall) { TupleEntry entry = sinkCall.getOutgoingEntry(); Fields fields = entry.getFields(); Tuple tuple = entry.getTuple(); if (fields.hasTypes()) { Type types[] = new Type[fields.size()]; for (int index = 0; index < fields.size(); index++) { Type type = fields.getType(index); if (type instanceof CoercibleType<?>) { types[index] = String.class; } else { types[index] = type; } } tuple = entry.getCoercedTuple(types); } return tuple; } }
@SuppressWarnings("unchecked") @Override public Result write(SinkCall<Object[], ?> sinkCall, Generator generator) { Tuple tuple = CascadingUtils.coerceToString(sinkCall); // consider names (in case of aliases these are already applied) List<String> names = (List<String>) sinkCall.getContext()[SINK_CTX_ALIASES]; generator.writeBeginObject(); for (int i = 0; i < tuple.size(); i++) { String name = (i < names.size() ? names.get(i) : "tuple" + i); // filter out fields if (shouldKeep(generator.getParentPath(), name)) { generator.writeFieldName(name); Object object = tuple.getObject(i); Result result = jdkWriter.write(object, generator); if (!result.isSuccesful()) { if (object instanceof Writable) { return writableWriter.write((Writable) object, generator); } return Result.FAILED(object); } } } generator.writeEndObject(); return Result.SUCCESFUL(); }
@Override public void sink( FlowProcess<? extends Configuration> flowProcess, SinkCall<Void, OutputCollector> sinkCall ) throws IOException { sinkCall.getOutput().collect( Tuple.NULL, sinkCall.getOutgoingEntry().getTuple() ); }
@Override public void sink(FlowProcess<JobConf> flowProcess, SinkCall<Void, OutputCollector> sinkCall) throws IOException { TupleEntry tupleEntry = sinkCall.getOutgoingEntry(); byte[] key = (byte[]) tupleEntry.getObject(0); byte[] val = (byte[]) tupleEntry.getObject(1); sinkCall.getOutput().collect(new BytesWritable(key), new BytesWritable(val)); } }
@SuppressWarnings("unchecked") @Override public void sink(FlowProcess<? extends JobConf> fp, SinkCall<Object[], OutputCollector> sc) throws IOException { TupleEntry tuple = sc.getOutgoingEntry(); if (tuple.size() != 1) { throw new RuntimeException("ParquetValueScheme expects tuples with an arity of exactly 1, but found " + tuple.getFields()); } T value = (T) tuple.getObject(0); OutputCollector output = sc.getOutput(); output.collect(null, value); }
@Override public void sink(FlowProcess<? extends JobConf> fp, SinkCall<Object[], OutputCollector> sink) throws IOException { TupleEntry tuple = sink.getOutgoingEntry(); OutputCollector outputCollector = sink.getOutput(); outputCollector.collect(null, tuple); } }
protected void writeHeader( SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { Fields fields = sinkCall.getOutgoingEntry().getFields(); Text text = (Text) sinkCall.getContext()[ 0 ]; StringBuilder line = (StringBuilder) sinkCall.getContext()[ 1 ]; Charset charset = (Charset) sinkCall.getContext()[ 2 ]; line = (StringBuilder) delimitedParser.joinFirstLine( fields, line ); text.set( line.toString().getBytes( charset ) ); sinkCall.getOutput().collect( null, text ); line.setLength( 0 ); }
@Override public void sink(FlowProcess<JobConf> flowProcess, SinkCall<Object[], OutputCollector> sinkCall) throws IOException { TupleEntry tupleEntry = sinkCall.getOutgoingEntry(); OutputCollector outputCollector = sinkCall.getOutput(); Tuple key = tupleEntry.selectTuple(keyField); ImmutableBytesWritable keyBytes = (ImmutableBytesWritable) key.getObject(0); Put put = new Put(keyBytes.get()); for (int i = 0; i < valueFields.length; i++) { Fields fieldSelector = valueFields[i]; TupleEntry values = tupleEntry.selectEntry(fieldSelector); for (int j = 0; j < values.getFields().size(); j++) { Fields fields = values.getFields(); Tuple tuple = values.getTuple(); ImmutableBytesWritable valueBytes = (ImmutableBytesWritable) tuple.getObject(j); put.add(Bytes.toBytes(familyNames[i]), Bytes.toBytes((String) fields.get(j)), valueBytes.get()); } } outputCollector.collect(null, put); }
@Override public void sink( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { TupleEntry tupleEntry = sinkCall.getOutgoingEntry(); Text text = (Text) sinkCall.getContext()[ 0 ]; StringBuilder line = (StringBuilder) sinkCall.getContext()[ 1 ]; Charset charset = (Charset) sinkCall.getContext()[ 2 ]; Iterable<String> strings = tupleEntry.asIterableOf( String.class ); line = (StringBuilder) delimitedParser.joinLine( strings, line ); text.set( line.toString().getBytes( charset ) ); sinkCall.getOutput().collect( null, text ); line.setLength( 0 ); }
@Override public void sinkPrepare( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) { OutputStream originalOutput = sinkCall.getOutput(); sinkCall.setContext( createOutput( originalOutput ) ); if( writeHeader && !isAppendingFile( sinkCall, originalOutput ) ) { Fields fields = sinkCall.getOutgoingEntry().getFields(); delimitedParser.joinFirstLine( fields, sinkCall.getContext() ); sinkCall.getContext().println(); } }
@Override public void sink( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) throws IOException { sinkCall.getContext().println( sinkCall.getOutgoingEntry().getTuple().toString() ); }
static Tuple coerceToString(SinkCall<?, ?> sinkCall) { return sinkCall.getOutgoingEntry().getTuple(); } }
@Override public void sink( FlowProcess<? extends Configuration> flowProcess, SinkCall<Object[], OutputCollector> sinkCall ) throws IOException { Fields found = sinkCall.getOutgoingEntry().getFields(); if( !found.equals( expectedFields ) ) throw new RuntimeException( "fields to not match, expect: " + expectedFields + ", found: " + found ); super.sink( flowProcess, sinkCall ); } }
@SuppressWarnings({ "rawtypes" }) @Override protected Object extractField(Object target) { List<String> fieldNames = getFieldNames(); for (int i = 0; i < fieldNames.size(); i++) { if (target instanceof SinkCall) { target = ((SinkCall) target).getOutgoingEntry().getObject(fieldNames.get(i)); if (target == null) { return NOT_FOUND; } } else { return NOT_FOUND; } } return target; }
/** {@inheritDoc} */ @Override public void sink(FlowProcess<? extends Properties> flowProcess, SinkCall<Void, List<Tuple>> sinkCall) throws IOException { sinkCall.getOutput().add(sinkCall.getOutgoingEntry().getTupleCopy()); }
@Override public void sink( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) throws IOException { TupleEntry tupleEntry = sinkCall.getOutgoingEntry(); Iterable<String> strings = tupleEntry.asIterableOf( String.class ); delimitedParser.joinLine( strings, sinkCall.getContext() ); sinkCall.getContext().println(); }
@Override public void sinkCleanup( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) throws IOException { sinkCall.getContext().flush(); sinkCall.setContext( null ); }
@Override public void sinkPrepare( FlowProcess<? extends Properties> flowProcess, SinkCall<PrintWriter, OutputStream> sinkCall ) throws IOException { sinkCall.setContext( createOutput( sinkCall.getOutput() ) ); }
@Override public String toString(Object field) { if (field instanceof SinkCall) { return ((SinkCall) field).getOutgoingEntry().toString(); } return field.toString(); } }