@Override public int getResponseSize() { int size = 2 + 4; for(Version v: results) { size += 4 + ((VectorClock) v).sizeInBytes(); } return size; }
public byte[] toBytes() { byte[] serialized = new byte[sizeInBytes()]; toBytes(serialized, 0); return serialized; }
@Override public String getDebugMessage() { return "Operation PUT " + ClientRequestHandler.getDebugMessageForKey(key) + " ValueHash" + (value == null ? "null" : value.hashCode()) + " ClockSize " + clock.sizeInBytes() + " ValueSize " + (value == null ? "null" : value.length); }
@Override public String getDebugMessage() { return "Operation DELETE " + ClientRequestHandler.getDebugMessageForKey(key) + " ClockSize " + version.sizeInBytes(); }
/** * Store the versioned values * * @param values list of versioned bytes * @return the list of versioned values rolled into an array of bytes */ private byte[] assembleValues(List<Versioned<byte[]>> values) throws IOException { ByteArrayOutputStream stream = new ByteArrayOutputStream(); DataOutputStream dataStream = new DataOutputStream(stream); for(Versioned<byte[]> value: values) { byte[] object = value.getValue(); dataStream.writeInt(object.length); dataStream.write(object); VectorClock clock = (VectorClock) value.getVersion(); dataStream.writeInt(clock.sizeInBytes()); dataStream.write(clock.toBytes()); } return stream.toByteArray(); }
public static int getResultsSize(List<Versioned<byte[]>> values) { int size = 4; for(Versioned<byte[]> v: values) { size += 4; size += ((VectorClock) v.getVersion()).sizeInBytes(); size += v.getValue().length; } return size; }
static int valueSize(Versioned<byte[]> value) { return value.getValue().length + ((VectorClock) value.getVersion()).sizeInBytes() + 1; }
private List<Versioned<byte[]>> readResults(DataInputStream inputStream) throws IOException { int resultSize = inputStream.readInt(); List<Versioned<byte[]>> results = new ArrayList<Versioned<byte[]>>(resultSize); for(int i = 0; i < resultSize; i++) { int valueSize = inputStream.readInt(); byte[] bytes = new byte[valueSize]; ByteUtils.read(inputStream, bytes); VectorClock clock = new VectorClock(bytes); results.add(new Versioned<byte[]>(ByteUtils.copy(bytes, clock.sizeInBytes(), bytes.length), clock)); } return results; }
public void writeDeleteRequest(DataOutputStream outputStream, String storeName, ByteArray key, VectorClock version, RequestRoutingType routingType) throws IOException { StoreUtils.assertValidKey(key); outputStream.writeByte(VoldemortOpCode.DELETE_OP_CODE); outputStream.writeUTF(storeName); outputStream.writeBoolean(routingType.equals(RequestRoutingType.ROUTED)); if(protocolVersion > 1) { outputStream.writeByte(routingType.getRoutingTypeCode()); } outputStream.writeInt(key.length()); outputStream.write(key.get()); VectorClock clock = version; outputStream.writeShort(clock.sizeInBytes()); outputStream.write(clock.toBytes()); }
public void writePutRequest(DataOutputStream outputStream, String storeName, ByteArray key, byte[] value, byte[] transforms, VectorClock version, RequestRoutingType routingType) throws IOException { StoreUtils.assertValidKey(key); outputStream.writeByte(VoldemortOpCode.PUT_OP_CODE); outputStream.writeUTF(storeName); outputStream.writeBoolean(routingType.equals(RequestRoutingType.ROUTED)); if(protocolVersion > 1) { outputStream.writeByte(routingType.getRoutingTypeCode()); } outputStream.writeInt(key.length()); outputStream.write(key.get()); outputStream.writeInt(value.length + version.sizeInBytes()); outputStream.write(version.toBytes()); outputStream.write(value); if(protocolVersion > 2) { if(transforms != null) { outputStream.writeBoolean(true); outputStream.writeInt(transforms.length); outputStream.write(transforms); } else outputStream.writeBoolean(false); } }
@Override public boolean parseRequest(DataInputStream inputStream) throws IOException { key = ClientRequestHandler.readKey(inputStream); int valueSize = inputStream.readInt(); clock = VectorClock.createVectorClock(inputStream); int vectorClockSize = clock.sizeInBytes(); value = new byte[valueSize - vectorClockSize]; ByteUtils.read(inputStream, value); transforms = ClientRequestHandler.readSingleTransform(inputStream, protocolVersion); return false; }
@Override public int getExpectedPutRequestSize(String storeName, ByteArray key, byte[] value, byte[] transforms, VectorClock version, RequestRoutingType routingType) { int size = 1 + ByteUtils.getUTFMaxLength(storeName) + 1; if(protocolVersion > 1) { size += 1; } size += (4 + key.length()); size += 4 + (value.length + version.sizeInBytes()); if(protocolVersion > 2) { size += 1; if(transforms != null) { size += 4 + transforms.length; } } return size; }
public Versioned<T> toObject(byte[] bytes) { VectorClock vectorClock = getVectorClock(bytes); int size = 1; if(vectorClock != null) size = vectorClock.sizeInBytes(); T t = innerSerializer.toObject(ByteUtils.copy(bytes, size, bytes.length)); return new Versioned<T>(t, vectorClock); }
public static List<Versioned<byte[]>> fromByteArray(byte[] bytes) { if(bytes.length < 1) throw new VoldemortException("Invalid value length: " + bytes.length); if(bytes[0] != VERSION) throw new VoldemortException("Unexpected version number in value: " + bytes[0]); int pos = 1; List<Versioned<byte[]>> vals = new ArrayList<Versioned<byte[]>>(2); while(pos < bytes.length) { VectorClock clock = new VectorClock(bytes, pos); pos += clock.sizeInBytes(); int valueSize = ByteUtils.readInt(bytes, pos); pos += ByteUtils.SIZE_OF_INT; byte[] val = new byte[valueSize]; System.arraycopy(bytes, pos, val, 0, valueSize); pos += valueSize; vals.add(Versioned.value(val, clock)); } if(pos != bytes.length) throw new VoldemortException((bytes.length - pos) + " straggling bytes found in value (this should not be possible)!"); return vals; }
public int toBytes(byte[] buf, int offset) { // write the number of versions ByteUtils.writeShort(buf, (short) versionMap.size(), offset); offset += ByteUtils.SIZE_OF_SHORT; // write the size of each version in bytes byte versionSize = ByteUtils.numberOfBytesRequired(getMaxVersion()); buf[offset] = versionSize; offset++; int clockEntrySize = ByteUtils.SIZE_OF_SHORT + versionSize; SortedSet<Short> nodeIds = versionMap.navigableKeySet(); for(Short nodeId: nodeIds) { Long version = versionMap.get(nodeId); ByteUtils.writeShort(buf, nodeId, offset); ByteUtils.writeBytes(buf, version, offset + ByteUtils.SIZE_OF_SHORT, versionSize); offset += clockEntrySize; } ByteUtils.writeLong(buf, this.timestamp, offset); return sizeInBytes(); }
/** * Returns the approximate size of slop to help in throttling * * @param slopVersioned The versioned slop whose size we want * @return Size in bytes */ private int slopSize(Versioned<Slop> slopVersioned) { int nBytes = 0; Slop slop = slopVersioned.getValue(); nBytes += slop.getKey().length(); nBytes += ((VectorClock) slopVersioned.getVersion()).sizeInBytes(); switch(slop.getOperation()) { case PUT: { nBytes += slop.getValue().length; break; } case DELETE: { break; } default: logger.error("Unknown slop operation: " + slop.getOperation()); } return nBytes; }
clock.sizeInBytes(), valueEntry.getData().length); Versioned<byte[]> value = new Versioned<byte[]>(bytes, clock);
public static byte[] toByteArray(List<Versioned<byte[]>> values) { int size = 1; for(Versioned<byte[]> v: values) { size += ((VectorClock) v.getVersion()).sizeInBytes(); size += ByteUtils.SIZE_OF_INT; size += v.getValue().length; } byte[] bytes = new byte[size]; int pos = 1; bytes[0] = VERSION; for(Versioned<byte[]> v: values) { pos += ((VectorClock) v.getVersion()).toBytes(bytes, pos); int len = v.getValue().length; ByteUtils.writeInt(bytes, len, pos); pos += ByteUtils.SIZE_OF_INT; System.arraycopy(v.getValue(), 0, bytes, pos, len); pos += len; } if(pos != bytes.length) throw new VoldemortException((bytes.length - pos) + " straggling bytes found in value (this should not be possible)!"); return bytes; }
clock.sizeInBytes(), valueEntry.getData().length); Versioned<byte[]> value = new Versioned<byte[]>(bytes, clock);
private DataSetStats calculateStats(StorageEngine<ByteArray, byte[], byte[]> store) { DataSetStats stats = new DataSetStats(); ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iter = store.entries(); try { int count = 0; while(iter.hasNext()) { Pair<ByteArray, Versioned<byte[]>> pair = iter.next(); VectorClock clock = (VectorClock) pair.getSecond().getVersion(); stats.countEntry(pair.getFirst().length(), pair.getSecond().getValue().length + clock.sizeInBytes()); if(count % 10000 == 0) logger.debug("Processing key " + count); count++; } } finally { iter.close(); } return stats; }