private void writeObject(ObjectOutputStream out) throws IOException { // serialize the parent fields and the final fields out.defaultWriteObject(); // the job conf knows how to serialize itself jobConf.write(out); // write the input split hadoopInputSplit.write(out); }
@Override public void write(DataOutput out) throws IOException { out.writeUTF(inputSplit.getClass().getName()); inputSplit.write(out); out.writeUTF(inputFormatClassName); }
@Override public void write(DataOutput out) throws IOException { assert (inputSplits != null && inputSplits.length > 0); out.writeUTF(inputSplits[0].getClass().getName()); out.writeInt(inputSplits.length); for (InputSplit inputSplit : inputSplits) { inputSplit.write(out); } out.writeUTF(inputFormatClassName); } }
@Override public void write(DataOutput out) throws IOException { out.writeUTF(inputSplit.getClass().getName()); inputSplit.write(out); out.writeUTF(inputFormatClassName); }
@Override public void write(DataOutput out) throws IOException { assert (inputSplits != null && inputSplits.length > 0); out.writeUTF(inputSplits[0].getClass().getName()); out.writeInt(inputSplits.length); for (InputSplit inputSplit : inputSplits) { inputSplit.write(out); } out.writeUTF(inputFormatClassName); } }
public void write(DataOutput out) throws IOException { Text.writeString(out, path.toString()); delegate.write(out); }
@Override public void write(DataOutput out) throws IOException { super.write(out); out.writeBoolean(isTableSplit); if (isTableSplit) { tableSplit.write(out); } else { snapshotSplit.write(out); } }
/** * Serializes each input split to string using Base64 encoding. * * @return list of serialized input splits */ public List<String> serialize() throws IOException { List<String> serializedInputSplits = new ArrayList<>(); for (InputSplit inputSplit : inputSplits) { final ByteArrayDataOutput byteArrayOutputStream = ByteStreams.newDataOutput(); inputSplit.write(byteArrayOutputStream); final String encoded = Base64.encodeBase64String(byteArrayOutputStream.toByteArray()); logger.debug("Encoded split string for split {} : {}", inputSplit, encoded); serializedInputSplits.add(encoded); } return serializedInputSplits; }
public void write(DataOutput out) throws IOException { Text.writeString(out, inputSplitClass.getName()); inputSplit.write(out); Text.writeString(out, inputFormatClass.getName()); Text.writeString(out, mapperClass.getName()); Text.writeString(out, schema.toString()); }
@Override public void process(Object[] arguments) throws HiveException { String query = stringOI.getPrimitiveJavaObject(arguments[0]); int num = intOI.get(arguments[1]); PlanFragment fragment = createPlanFragment(query, num); TezWork tezWork = fragment.work; Schema schema = fragment.schema; try { for (InputSplit s : getSplits(jc, num, tezWork, schema)) { Object[] os = new Object[1]; bos.reset(); s.write(dos); byte[] frozen = bos.toByteArray(); os[0] = frozen; forward(os); } } catch (Exception e) { throw new HiveException(e); } }
Object[] os = new Object[1]; bos.reset(); s.write(dos); byte[] frozen = bos.toByteArray(); os[0] = frozen;
private void writeObject(ObjectOutputStream out) throws IOException { out.writeInt(index); out.writeUTF(split.getClass().getName()); split.write(out); }
@Override public void write(DataOutput out) throws IOException { out.writeUTF(inputSplit.getClass().getName()); inputSplit.write(out); out.writeUTF(inputFormatClassName); }
@Override public void write(DataOutput out) throws IOException { assert (inputSplits != null && inputSplits.length > 0); out.writeUTF(inputSplits[0].getClass().getName()); out.writeInt(inputSplits.length); for (InputSplit inputSplit : inputSplits) { inputSplit.write(out); } out.writeUTF(inputFormatClassName); } }
@Override public void write(DataOutput out) throws IOException { assert (inputSplits != null && inputSplits.length > 0); out.writeUTF(inputSplits[0].getClass().getName()); out.writeInt(inputSplits.length); for (InputSplit inputSplit : inputSplits) { inputSplit.write(out); } out.writeUTF(inputFormatClassName); } }
public void write( DataOutput out ) throws IOException { out.writeUTF( inputSplit.getClass().getName() ); String[] keys = config.keySet().toArray( new String[ config.size() ] ); String[] values = new String[ keys.length ]; for( int i = 0; i < keys.length; i++ ) values[ i ] = config.get( keys[ i ] ); WritableUtils.writeStringArray( out, keys ); WritableUtils.writeStringArray( out, values ); inputSplit.write( out ); }
public void write(DataOutput out) throws IOException { Text.writeString(out, inputSplitClass.getName()); inputSplit.write(out); Text.writeString(out, inputFormatClass.getName()); Text.writeString(out, mapperClass.getName()); }
public void write(DataOutput out) throws IOException { Text.writeString(out, inputSplitClass.getName()); inputSplit.write(out); Text.writeString(out, inputFormatClass.getName()); Text.writeString(out, mapperClass.getName()); }
public void write(DataOutput out) throws IOException { Text.writeString(out, inputSplitClass.getName()); inputSplit.write(out); Text.writeString(out, inputFormatClass.getName()); Text.writeString(out, mapperClass.getName()); }
private SerializedInputSplit serialize(InputSplit split) throws IOException{ ByteArrayDataOutput output = ByteStreams.newDataOutput(); split.write(output); return SerializedInputSplit.newBuilder() .setInputSplitClass(split.getClass().getName()) .setInputSplit(com.google.protobuf.ByteString.copyFrom(output.toByteArray())).build(); }