@Override protected GenericRecord nextAvroRecord() throws IOException { if (decoder.isEnd()) { return null; } try { genericRecord = datumReader.read(genericRecord, decoder); } catch (final EOFException eof) { return null; } return genericRecord; }
byte[] magic = new byte[DataFileConstants.MAGIC.length]; try { vin.readFixed(magic); // read magic } catch (IOException e) { throw new IOException("Not a data file.", e); throw new IOException("Not a data file."); long l = vin.readMapStart(); // read meta data if (l > 0) { do { for (long i = 0; i < l; i++) { String key = vin.readString(null).toString(); ByteBuffer value = vin.readBytes(null); byte[] bb = new byte[value.remaining()]; value.get(bb); header.metaKeyList.add(key); } while ((l = vin.mapNext()) != 0); vin.readFixed(header.sync); // read sync
/** Close this reader. */ @Override public void close() throws IOException { vin.inputStream().close(); }
private static int hashBytes(int init, HashData data, int len, boolean rev) throws IOException { int hashCode = init; byte[] bytes = data.decoder.getBuf(); int start = data.decoder.getPos(); int end = start+len; if (rev) for (int i = end-1; i >= start; i--) hashCode = hashCode*31 + bytes[i]; else for (int i = start; i < end; i++) hashCode = hashCode*31 + bytes[i]; data.decoder.skipFixed(len); return hashCode; }
/** * Reads the count of items in the current array or map and skip those items, * if possible. If it could skip the items, keep repeating until there are no * more items left in the array or map. If items cannot be skipped (because * byte count to skip is not found in the stream) return the count of the * items found. The client needs to skip the items individually. * * @return Zero if there are no more items to skip and end of array/map is * reached. Positive number if some items are found that cannot be * skipped and the client needs to skip them individually. * @throws IOException */ private long doSkipItems() throws IOException { long result = readInt(); while (result < 0) { long bytecount = readLong(); doSkipBytes(bytecount); result = readInt(); } return result; }
boolean hasNextBlock() { try { if (availableBlock) return true; if (vin.isEnd()) return false; blockRemaining = vin.readLong(); // read block count blockSize = vin.readLong(); // read block size if (blockSize > Integer.MAX_VALUE || blockSize < 0) { throw new IOException("Block size invalid or too large for this " + "implementation: " + blockSize); } blockCount = blockRemaining; availableBlock = true; return true; } catch (EOFException eof) { return false; } catch (IOException e) { throw new AvroRuntimeException(e); } }
private void skipSync() throws IOException { vin.readFixed(syncBuffer); if (!Arrays.equals(syncBuffer, sync)) throw new IOException("Invalid sync!"); }
@Test public void testInputStreamPartiallyUsed() throws IOException { BinaryDecoder bd = factory.binaryDecoder( new ByteArrayInputStream(data), null); InputStream test = bd.inputStream(); InputStream check = new ByteArrayInputStream(data); // triggers buffer fill if unused and tests isEnd() try { Assert.assertFalse(bd.isEnd()); } catch (UnsupportedOperationException e) { // this is ok if its a DirectBinaryDecoder. if (bd.getClass() != DirectBinaryDecoder.class) { throw e; } } bd.readFloat(); // use data, and otherwise trigger buffer fill check.skip(4); // skip the same # of bytes here validateInputStreamReads(test, check); }
public TetherData deserialize(TetherData datum) throws IOException { if (datum == null) datum = new TetherData(); datum.buffer(decoder.readBytes(datum.buffer())); return datum; }
InputStream in = vin.inputStream(); vin.readFixed(syncBuffer); do { int j = 0;
@Override public String readString() throws IOException { return readString(scratchUtf8).toString(); }
/** * Returns the number of items to follow in the current array or map. Returns * 0 if there are no more items in the current array and the array/map has * ended. * * @throws IOException */ protected long doReadItemCount() throws IOException { long result = readLong(); if (result < 0) { readLong(); // Consume byte-count if present result = -result; } return result; }
@Override public Object parse(InputStream stream) { try { if (message.isOneWay()) return null; BinaryDecoder in = DECODER_FACTORY.binaryDecoder(stream, null); if (!in.readBoolean()) { Object response = new SpecificDatumReader(message.getResponse()).read(null, in); return response; } else { Object value = new SpecificDatumReader(message.getErrors()).read(null, in); if (value instanceof Exception) { return value; } return new AvroRuntimeException(value.toString()); } } catch (IOException e) { throw Status.INTERNAL.withCause(e). withDescription("Error deserializing avro response").asRuntimeException(); } finally { AvroGrpcUtils.skipAndCloseQuietly(stream); } }
@Override public long readMapStart() throws IOException { return wrappedDecoder.readMapStart(); }
@Override public long mapNext() throws IOException { return wrappedDecoder.mapNext(); }
@Override public void skipFloat() throws IOException { // floats have fixed length of 4 bytes _decoder.skipFixed(4); }
@Override public Utf8 readString(Utf8 old) throws IOException { long length = readLong(); if (length > MAX_ARRAY_SIZE) { throw new UnsupportedOperationException("Cannot read strings longer than " + MAX_ARRAY_SIZE + " bytes"); } if (length < 0L) { throw new AvroRuntimeException("Malformed data. Length is negative: " + length); } Utf8 result = (old != null ? old : new Utf8()); result.setByteLength((int) length); if (0L != length) { doReadBytes(result.getBytes(), 0, (int) length); } return result; }
private static int hashBytes(int init, HashData data, int len, boolean rev) throws IOException { int hashCode = init; byte[] bytes = data.decoder.getBuf(); int start = data.decoder.getPos(); int end = start+len; if (rev) for (int i = end-1; i >= start; i--) hashCode = hashCode*31 + bytes[i]; else for (int i = start; i < end; i++) hashCode = hashCode*31 + bytes[i]; data.decoder.skipFixed(len); return hashCode; }
private void skipSync() throws IOException { vin.readFixed(syncBuffer); if (!Arrays.equals(syncBuffer, sync)) throw new IOException("Invalid sync!"); }
boolean hasNextBlock() { try { if (availableBlock) return true; if (vin.isEnd()) return false; blockRemaining = vin.readLong(); // read block count blockSize = vin.readLong(); // read block size if (blockSize > Integer.MAX_VALUE || blockSize < 0) { throw new IOException("Block size invalid or too large for this " + "implementation: " + blockSize); } blockCount = blockRemaining; availableBlock = true; return true; } catch (EOFException eof) { return false; } catch (IOException e) { throw new AvroRuntimeException(e); } }