@Override public void appendSetup(AppendSetup appendSetup) { log.info("Received AppendSetup {}", appendSetup); long ackLevel = appendSetup.getLastEventNumber(); ackUpTo(ackLevel); List<Append> toRetransmit = state.getAllInflight() .stream() .map(entry -> new Append(segmentName, writerId, entry.getKey(), 1, entry.getValue().getData(), null)) .collect(Collectors.toList()); ClientConnection connection = state.getConnection(); if (connection == null) { log.warn("Connection setup could not be completed because connection is already failed.", writerId); return; } if (toRetransmit == null || toRetransmit.isEmpty()) { log.info("Connection setup complete for writer {}", writerId); state.connectionSetupComplete(connection); } else { connection.sendAsync(toRetransmit, e -> { if (e == null) { state.connectionSetupComplete(connection); } else { failConnection(e); } }); } }
if (appends.get(0).isConditional()) { outstandingAppend = appends.remove(0); } else { for (Iterator<Append> iterator = appends.iterator(); iterator.hasNext(); ) { Append a = iterator.next(); if (a.isConditional()) { break; toAppend[i] = a.getData(); last = a; eventCount += a.getEventCount(); iterator.remove(); String segment = last.getSegment(); long eventNumber = last.getEventNumber(); outstandingAppend = new Append(segment, writer, eventNumber, eventCount, data, null);
private CompletableFuture<Void> storeAppend(Append append) { long lastEventNumber; synchronized (lock) { lastEventNumber = latestEventNumbers.get(Pair.of(append.getSegment(), append.getWriterId())); } List<AttributeUpdate> attributes = Arrays.asList( new AttributeUpdate(append.getWriterId(), AttributeUpdateType.ReplaceIfEquals, append.getEventNumber(), lastEventNumber), new AttributeUpdate(EVENT_COUNT, AttributeUpdateType.Accumulate, append.getEventCount())); ByteBuf buf = append.getData().asReadOnly(); byte[] bytes = new byte[buf.readableBytes()]; buf.readBytes(bytes); if (append.isConditional()) { return store.append(append.getSegment(), append.getExpectedLength(), bytes, attributes, TIMEOUT); } else { return store.append(append.getSegment(), bytes, attributes, TIMEOUT); } }
private void validateAppend(Append append, Session session) { if (session == null || !session.id.equals(append.getWriterId())) { throw new InvalidMessageException("Sending appends without setting up the append."); } if (append.getEventNumber() <= session.lastEventNumber) { throw new InvalidMessageException("Events written out of order. Received: " + append.getEventNumber() + " following: " + session.lastEventNumber); } if (append.isConditional()) { throw new IllegalArgumentException("Conditional appends should be written via a ConditionalAppend object."); } Preconditions.checkState(bytesLeftInBlock == 0 || bytesLeftInBlock > TYPE_PLUS_LENGTH_SIZE, "Bug in CommandEncoder.encode, block is too small."); }
@Override public void send(Append append) throws ConnectionFailedException { recentMessage.set(true); batchSizeTracker.recordAppend(append.getEventNumber(), append.getData().readableBytes()); Futures.getAndHandleExceptions(getChannel().writeAndFlush(append), ConnectionFailedException::new); }
long previousEventNumber; synchronized (lock) { previousEventNumber = latestEventNumbers.get(Pair.of(append.getSegment(), append.getWriterId())); Preconditions.checkState(outstandingAppend == append, "Synchronization error in: %s while processing append: %s.", if (conditionalFailed) { log.debug("Conditional append failed due to incorrect offset: {}, {}", append, exception.getMessage()); connection.send(new ConditionalCheckFailed(append.getWriterId(), append.getEventNumber())); } else { handleException(append.getWriterId(), append.getEventNumber(), append.getSegment(), "appending data", exception); if (statsRecorder != null && !StreamSegmentNameUtils.isTransactionSegment(append.getSegment())) { statsRecorder.record(append.getSegment(), append.getDataLength(), append.getEventCount()); final DataAppended dataAppendedAck = new DataAppended(append.getWriterId(), append.getEventNumber(), previousEventNumber); log.trace("Sending DataAppended : {}", dataAppendedAck); connection.send(dataAppendedAck); dynamicLogger.incCounterValue(globalMetricName(SEGMENT_WRITE_BYTES), append.getDataLength()); dynamicLogger.incCounterValue(globalMetricName(SEGMENT_WRITE_EVENTS), append.getEventCount()); if (!StreamSegmentNameUtils.isTransactionSegment(append.getSegment())) { dynamicLogger.incCounterValue(nameFromSegment(SEGMENT_WRITE_BYTES, append.getSegment()), append.getDataLength()); dynamicLogger.incCounterValue(nameFromSegment(SEGMENT_WRITE_EVENTS, append.getSegment()), append.getEventCount()); outstandingAppend = null; if (exception == null) {
/** * Append data to the store. * Because ordering dictates that there only be one outstanding append from a given connection, this is implemented * by adding the append to a queue. */ @Override public void append(Append append) { log.trace("Processing append received from client {}", append); UUID id = append.getWriterId(); synchronized (lock) { Long lastEventNumber = latestEventNumbers.get(Pair.of(append.getSegment(), id)); Preconditions.checkState(lastEventNumber != null, "Data from unexpected connection: %s.", id); Preconditions.checkState(append.getEventNumber() >= lastEventNumber, "Event was already appended."); waitingAppends.put(id, append); } pauseOrResumeReading(); performNextWrite(); }
/** * If there is too much data waiting throttle the producer by stopping consumption from the socket. * If there is room for more data, we resume consuming from the socket. */ private void pauseOrResumeReading() { int bytesWaiting; synchronized (lock) { bytesWaiting = waitingAppends.values() .stream() .mapToInt(a -> a.getData().readableBytes()) .sum(); } if (bytesWaiting > HIGH_WATER_MARK) { log.debug("Pausing writing from connection {}", connection); connection.pauseReading(); } if (bytesWaiting < LOW_WATER_MARK) { log.trace("Resuming writing from connection {}", connection); connection.resumeReading(); } }
private void verify(List<Object> results, int numValues, int sizeOfEachValue) { int currentValue = -1; int currentCount = sizeOfEachValue; for (Object r : results) { Append append = (Append) r; assertEquals("Append split mid event", sizeOfEachValue, currentCount); while (append.getData().isReadable()) { if (currentCount == sizeOfEachValue) { assertEquals(EVENT.getCode(), append.getData().readInt()); assertEquals(sizeOfEachValue, append.getData().readInt()); currentCount = 0; currentValue++; } byte readByte = append.getData().readByte(); assertEquals((byte) currentValue, readByte); currentCount++; } assertEquals(currentValue, append.getEventNumber()); } assertEquals(numValues - 1, currentValue); assertEquals(currentCount, sizeOfEachValue); }
/** * If there isn't already an append outstanding against the store, write a new one. * Appends are opportunistically batched here. i.e. If many are waiting they are combined into a single append and * that is written. */ private void performNextWrite() { Append append = getNextAppend(); if (append == null) { return; } long traceId = LoggerHelpers.traceEnter(log, "storeAppend", append); Timer timer = new Timer(); storeAppend(append) .whenComplete((v, e) -> { handleAppendResult(append, e); LoggerHelpers.traceLeave(log, "storeAppend", traceId, v, e); if (e == null) { WRITE_STREAM_SEGMENT.reportSuccessEvent(timer.getElapsed()); } else { WRITE_STREAM_SEGMENT.reportFailEvent(timer.getElapsed()); } }) .whenComplete((v, e) -> append.getData().release()); }
@Before public void setUp() throws Exception { when(buffer.readableBytes()).thenReturn(10); appendCmd = new Append("segment0", UUID.randomUUID(), 2, 1, buffer, 10L); doNothing().when(tracker).recordAppend(anyLong(), anyInt()); when(ctx.channel()).thenReturn(ch); when(ch.eventLoop()).thenReturn(loop); when(ch.writeAndFlush(any(Object.class))).thenReturn(completedFuture); handler = new ClientConnectionInboundHandler("testConnection", processor, tracker); }
@Override public void sendAsync(List<Append> appends, CompletedCallback callback) { recentMessage.set(true); Channel ch; try { ch = getChannel(); } catch (ConnectionFailedException e) { callback.complete(new ConnectionFailedException("Connection to " + connectionName + " is not established.")); return; } PromiseCombiner combiner = new PromiseCombiner(); for (Append append : appends) { batchSizeTracker.recordAppend(append.getEventNumber(), append.getData().readableBytes()); combiner.add(ch.write(append)); } ch.flush(); ChannelPromise promise = ch.newPromise(); promise.addListener(new GenericFutureListener<Future<? super Void>>() { @Override public void operationComplete(Future<? super Void> future) throws Exception { Throwable cause = future.cause(); callback.complete(cause == null ? null : new ConnectionFailedException(cause)); } }); combiner.finish(promise); }
private void testFlush(int size) throws Exception { @Cleanup("release") ByteBuf fakeNetwork = ByteBufAllocator.DEFAULT.buffer(); ArrayList<Object> received = setupAppend(streamName, writerId, fakeNetwork); append(streamName, writerId, size, 0, size, fakeNetwork); read(fakeNetwork, received); KeepAlive keepAlive = new KeepAlive(); encoder.encode(null, keepAlive, fakeNetwork); read(fakeNetwork, received); assertEquals(2, received.size()); Append one = (Append) received.get(0); assertEquals(size + TYPE_PLUS_LENGTH_SIZE, one.getData().readableBytes()); KeepAlive two = (KeepAlive) received.get(1); assertEquals(keepAlive, two); }
Append append = new Append(segmentName, writerId, eventNumber, 1, event.getData(), null); log.trace("Sending append request: {}", append); connection.send(append);
session.lastEventNumber = append.getEventNumber(); session.eventCount++; ByteBuf data = append.getData().slice(); int msgSize = data.readableBytes();
@Test public void testAppendAtBlockBound() throws Exception { int size = appendBlockSize; @Cleanup("release") ByteBuf fakeNetwork = ByteBufAllocator.DEFAULT.buffer(); ArrayList<Object> received = setupAppend(streamName, writerId, fakeNetwork); append(streamName, writerId, size, 1, size, fakeNetwork); read(fakeNetwork, received); assertEquals(1, received.size()); append(streamName, writerId, size + size / 2, 2, size / 2, fakeNetwork); read(fakeNetwork, received); assertEquals(1, received.size()); KeepAlive keepAlive = new KeepAlive(); encoder.encode(null, keepAlive, fakeNetwork); read(fakeNetwork, received); assertEquals(3, received.size()); Append one = (Append) received.get(0); Append two = (Append) received.get(1); assertEquals(size + TYPE_PLUS_LENGTH_SIZE, one.getData().readableBytes()); assertEquals(size / 2 + TYPE_PLUS_LENGTH_SIZE, two.getData().readableBytes()); KeepAlive three = (KeepAlive) received.get(2); assertEquals(keepAlive, three); }
private void sendAndVerifyEvent(UUID cid, ClientConnection connection, SegmentOutputStreamImpl output, ByteBuffer data, int num) throws SegmentSealedException, ConnectionFailedException { CompletableFuture<Void> acked = new CompletableFuture<>(); output.write(PendingEvent.withoutHeader(null, data, acked)); verify(connection).send(new Append(SEGMENT, cid, num, 1, Unpooled.wrappedBuffer(data), null)); assertEquals(false, acked.isDone()); }
private long append(String segment, UUID writerId, long offset, int messageNumber, int length, ByteBuf out) throws Exception { byte[] content = new byte[length]; Arrays.fill(content, (byte) messageNumber); Event event = new Event(Unpooled.wrappedBuffer(content)); Append msg = new Append(segment, writerId, messageNumber, event); assertEquals(length + WireCommands.TYPE_PLUS_LENGTH_SIZE, msg.data.readableBytes()); encoder.encode(null, msg, out); assertEquals(length + WireCommands.TYPE_PLUS_LENGTH_SIZE, msg.data.readableBytes()); return offset + length; }
@Test public void testSetupSkipped() { String streamSegmentName = "testAppendSegment"; UUID clientId = UUID.randomUUID(); byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 }; StreamSegmentStore store = mock(StreamSegmentStore.class); ServerConnection connection = mock(ServerConnection.class); AppendProcessor processor = new AppendProcessor(store, connection, new FailingRequestProcessor(), null); try { processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null)); fail(); } catch (RuntimeException e) { //expected } verifyNoMoreInteractions(connection); verifyNoMoreInteractions(store); }
@Test public void testInvalidOffset() { String streamSegmentName = "testAppendSegment"; UUID clientId = UUID.randomUUID(); byte[] data = new byte[] { 1, 2, 3, 4, 6, 7, 8, 9 }; StreamSegmentStore store = mock(StreamSegmentStore.class); ServerConnection connection = mock(ServerConnection.class); AppendProcessor processor = new AppendProcessor(store, connection, new FailingRequestProcessor(), null); setupGetAttributes(streamSegmentName, clientId, 100, store); processor.setupAppend(new SetupAppend(1, clientId, streamSegmentName, "")); try { processor.append(new Append(streamSegmentName, clientId, data.length, 1, Unpooled.wrappedBuffer(data), null)); fail(); } catch (RuntimeException e) { //expected } verify(store).getAttributes(anyString(), eq(Collections.singleton(clientId)), eq(true), eq(AppendProcessor.TIMEOUT)); verify(connection).send(new AppendSetup(1, streamSegmentName, clientId, 100)); verify(connection, atLeast(0)).resumeReading(); verifyNoMoreInteractions(connection); verifyNoMoreInteractions(store); }