/** Used by child copy constructors. */ protected synchronized void copy(Writable other) { if (other != null) { try { DataOutputBuffer out = new DataOutputBuffer(); other.write(out); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), out.getLength()); readFields(in); } catch (IOException e) { throw new IllegalArgumentException("map cannot be copied: " + e.getMessage()); } } else { throw new IllegalArgumentException("source map cannot be null"); } }
/** Workhorse to check and write out compressed data/lengths */ private synchronized void writeBuffer(DataOutputBuffer uncompressedDataBuffer) throws IOException { deflateFilter.resetState(); buffer.reset(); deflateOut.write(uncompressedDataBuffer.getData(), 0, uncompressedDataBuffer.getLength()); deflateOut.flush(); deflateFilter.finish(); WritableUtils.writeVInt(out, buffer.getLength()); out.write(buffer.getData(), 0, buffer.getLength()); }
/** Read a compressed buffer */ private synchronized void readBuffer(DataInputBuffer buffer, CompressionInputStream filter) throws IOException { // Read data into a temporary buffer DataOutputBuffer dataBuffer = new DataOutputBuffer(); try { int dataBufferLength = WritableUtils.readVInt(in); dataBuffer.write(in, dataBufferLength); // Set up 'buffer' connected to the input-stream buffer.reset(dataBuffer.getData(), 0, dataBuffer.getLength()); } finally { dataBuffer.close(); } // Reset the codec filter.resetState(); }
/** * Serialize the shuffle port into a ByteBuffer for use later on. * @param port the port to be sent to the ApplciationMaster * @return the serialized form of the port. */ public static ByteBuffer serializeMetaData(int port) throws IOException { //TODO these bytes should be versioned DataOutputBuffer port_dob = new DataOutputBuffer(); port_dob.writeInt(port); return ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength()); }
public static void setTokensFor(ContainerLaunchContext amContainer, List<Path> paths, Configuration conf) throws IOException { Credentials credentials = new Credentials(); // for HDFS TokenCache.obtainTokensForNamenodes(credentials, paths.toArray(new Path[0]), conf); // for HBase obtainTokenForHBase(credentials, conf); // for user UserGroupInformation currUsr = UserGroupInformation.getCurrentUser(); Collection<Token<? extends TokenIdentifier>> usrTok = currUsr.getTokens(); for (Token<? extends TokenIdentifier> token : usrTok) { final Text id = new Text(token.getIdentifier()); LOG.info("Adding user token " + id + " with " + token); credentials.addToken(id, token); } try (DataOutputBuffer dob = new DataOutputBuffer()) { credentials.writeTokenStorageToStream(dob); if (LOG.isDebugEnabled()) { LOG.debug("Wrote tokens. Credentials buffer length: " + dob.getLength()); } ByteBuffer securityTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); amContainer.setTokens(securityTokens); } }
/** * Generate a string with the url-quoted base64 encoded serialized form * of the Writable. * @param obj the object to serialize * @return the encoded string * @throws IOException */ private static String encodeWritable(Writable obj) throws IOException { DataOutputBuffer buf = new DataOutputBuffer(); obj.write(buf); Base64 encoder = new Base64(0, null, true); byte[] raw = new byte[buf.getLength()]; System.arraycopy(buf.getData(), 0, raw, 0, buf.getLength()); return encoder.encodeToString(raw); }
private ByteBuffer getSecurityTokens() throws IOException { Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); Closer closer = Closer.create(); try { DataOutputBuffer dataOutputBuffer = closer.register(new DataOutputBuffer()); credentials.writeTokenStorageToStream(dataOutputBuffer); // Remove the AM->RM token so that containers cannot access it Iterator<Token<?>> tokenIterator = credentials.getAllTokens().iterator(); while (tokenIterator.hasNext()) { Token<?> token = tokenIterator.next(); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { tokenIterator.remove(); } } return ByteBuffer.wrap(dataOutputBuffer.getData(), 0, dataOutputBuffer.getLength()); } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } }
@Override public boolean next(LongWritable key, Text value) throws IOException { if (readUntilMatch(this.startTag, false)) { this.recordStartPos = this.pos - this.startTag.length; try { this.buffer.write(this.startTag); if (readUntilMatch(this.endTag, true)) { key.set(this.recordStartPos); value.set(this.buffer.getData(), 0, this.buffer.getLength()); return true; } } finally { this.buffer.reset(); } } return false; }
@Override protected void map(Writable key, Writable value, Context context) throws IOException, InterruptedException { BehemothDocument doc = new BehemothDocument(); doc.setUrl(key.toString()); // TODO: Is this the right way to do this? We need the bytes. DataOutputBuffer out = new DataOutputBuffer(); value.write(out); doc.setContent(out.getData()); // doc.setContent(value.toString().getBytes(Charset.forName("UTF-8"))); context.write(new Text(key.toString()), doc); } }
if (key.getClass() != getKeyClass()) throw new IOException("wrong key class: "+key.getClass().getName() +" is not "+keyClass); outBuf.reset(); keyLength = next(outBuf); if (keyLength < 0) return false; valBuffer.reset(outBuf.getData(), outBuf.getLength()); key.readFields(valBuffer); valBuffer.mark(0); if (valBuffer.getPosition() != keyLength) throw new IOException(key + " read " + valBuffer.getPosition() + " bytes, should read " + keyLength); readBlock(); } catch (EOFException eof) { return false; int keyLength = WritableUtils.readVInt(keyLenIn); key.readFields(keyIn); --noBufferedKeys;
BytesWritable bw = (BytesWritable) writableCol; byte[] bytes = bw.getBytes(); int start = buffer.getLength(); int length = bw.getLength(); try { buffer.write(bytes, 0, length); } catch (IOException ioe) { throw new IllegalStateException("bad write", ioe); bcv.setRef(rowIndex, buffer.getData(), start, length); } else { setNullColIsNullValue(bcv, rowIndex); bcv.isNull[rowIndex] = false; Text colText = (Text) writableCol; int start = buffer.getLength(); int length = colText.getLength(); try { buffer.write(colText.getBytes(), 0, length); } catch (IOException ioe) { throw new IllegalStateException("bad write", ioe); bcv.setRef(rowIndex, buffer.getData(), start, length); } else { setNullColIsNullValue(bcv, rowIndex); int start = buffer.getLength(); try {
/** * A helper function to serialize the JobTokenIdentifier to be sent to the * ShuffleHandler as ServiceData. * @param jobToken the job token to be used for authentication of * shuffle data requests. * @return the serialized version of the jobToken. */ public static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken) throws IOException { //TODO these bytes should be versioned DataOutputBuffer jobToken_dob = new DataOutputBuffer(); jobToken.write(jobToken_dob); return ByteBuffer.wrap(jobToken_dob.getData(), 0, jobToken_dob.getLength()); }
@Test public void testFSCheckpointIDSerialization() throws IOException { Path inpath = new Path("/tmp/blah"); FSCheckpointID cidin = new FSCheckpointID(inpath); DataOutputBuffer out = new DataOutputBuffer(); cidin.write(out); out.close(); FSCheckpointID cidout = new FSCheckpointID(null); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), 0, out.getLength()); cidout.readFields(in); in.close(); assert cidin.equals(cidout); }
public TextRecordInputStream(FileStatus f) throws IOException { final Path fpath = f.getPath(); final Configuration lconf = getConf(); r = new SequenceFile.Reader(lconf, SequenceFile.Reader.file(fpath)); key = ReflectionUtils.newInstance( r.getKeyClass().asSubclass(Writable.class), lconf); val = ReflectionUtils.newInstance( r.getValueClass().asSubclass(Writable.class), lconf); inbuf = new DataInputBuffer(); outbuf = new DataOutputBuffer(); }
public void write(DataOutput out) throws IOException { if (firstKey == null) { Utils.writeVInt(out, 0); return; } DataOutputBuffer dob = new DataOutputBuffer(); Utils.writeVInt(dob, firstKey.size()); dob.write(firstKey.buffer()); Utils.writeVInt(out, dob.size()); out.write(dob.getData(), 0, dob.getLength()); for (TFileIndexEntry entry : index) { dob.reset(); entry.write(dob); Utils.writeVInt(out, dob.getLength()); out.write(dob.getData(), 0, dob.getLength()); } } }
@Override public int read() throws IOException { int ret; if (null == inbuf || -1 == (ret = inbuf.read())) { if (!r.next(key, val)) { return -1; } byte[] tmp = key.toString().getBytes(StandardCharsets.UTF_8); outbuf.write(tmp, 0, tmp.length); outbuf.write('\t'); tmp = val.toString().getBytes(StandardCharsets.UTF_8); outbuf.write(tmp, 0, tmp.length); outbuf.write('\n'); inbuf.reset(outbuf.getData(), outbuf.getLength()); outbuf.reset(); ret = inbuf.read(); } return ret; }
/** Convert writables to a byte array */ public static byte[] toByteArray(Writable... writables) { final DataOutputBuffer out = new DataOutputBuffer(); try { for(Writable w : writables) { w.write(out); } out.close(); } catch (IOException e) { throw new RuntimeException("Fail to convert writables to a byte array",e); } return out.getData(); }