@Test public void overwriteThrowsException() throws Exception { byte[] testString = "hello world".getBytes(); client.write(0, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get(); assertThatThrownBy(() -> client.write(0, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get()) .isInstanceOf(ExecutionException.class) .hasCauseInstanceOf(OverwriteException.class); }
@Test public void canReadWrite() throws Exception { byte[] testString = "hello world".getBytes(); client.write(0, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get(); LogData r = client.read(0).get().getAddresses().get(0L); assertThat(r.getType()) .isEqualTo(DataType.DATA); assertThat(r.getPayload(new CorfuRuntime())) .isEqualTo(testString); }
private QuorumFuturesFactory.CompositeFuture<Boolean> getWriteFuture( RuntimeLayout runtimeLayout, ILogData data) { int numUnits = runtimeLayout.getLayout().getSegmentLength(data.getGlobalAddress()); long globalAddress = data.getGlobalAddress(); CompletableFuture<Boolean>[] futures = new CompletableFuture[numUnits]; for (int i = 0; i < numUnits; i++) { log.trace("Write[{}]: quorum {}/{}", globalAddress, i + 1, numUnits); futures[i] = runtimeLayout.getLogUnitClient(globalAddress, i).write(data); } QuorumFuturesFactory.CompositeFuture<Boolean> future = QuorumFuturesFactory.getQuorumFuture(Boolean::compareTo, futures, OverwriteException.class, DataOutrankedException.class); return future; }
@Test public void CorruptedDataReadThrowsException() throws Exception { byte[] testString = "hello world".getBytes(); client.write(0, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get(); client.write(StreamLogFiles.RECORDS_PER_LOG_FILE + 1, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get(); // Corrupt the written log entry String logDir = serverContext.getServerConfig().get("--log-path") + File.separator + "log"; String logFilePath = logDir + File.separator + "0.log"; RandomAccessFile file = new RandomAccessFile(logFilePath, "rw"); ByteBuffer metaDataBuf = ByteBuffer.allocate(METADATA_SIZE); file.getChannel().read(metaDataBuf); metaDataBuf.flip(); LogUnitServer server2 = new LogUnitServer(serverContext); serverRouter.reset(); serverRouter.addServer(server2); Types.Metadata metadata = Types.Metadata.parseFrom(metaDataBuf.array()); final int fileOffset = Integer.BYTES + METADATA_SIZE + metadata.getLength() + 20; final int CORRUPT_BYTES = 0xFFFF; file.seek(fileOffset); // Skip file header file.writeInt(CORRUPT_BYTES); file.close(); // Try to read a corrupted log entry assertThatThrownBy(() -> client.read(0).get()) .isInstanceOf(ExecutionException.class) .hasCauseInstanceOf(DataCorruptionException.class); }
@Test public void holeFillCannotBeOverwritten() throws Exception { byte[] testString = "hello world".getBytes(); final long address0 = 0; Token token = new Token(0, address0); client.fillHole(token).get(); LogData r = client.read(address0).get().getAddresses().get(0L); assertThat(r.getType()) .isEqualTo(DataType.HOLE); assertThat(r.getGlobalAddress()).isEqualTo(address0); assertThatThrownBy(() -> client.write(address0, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get()) .isInstanceOf(ExecutionException.class) .hasCauseInstanceOf(OverwriteException.class); }
/** * Testing that the clientId/ThreadId is persisted and that two * LogData entries are equal if they have the same runtime and are coming * from the same thread. * @throws Exception */ @Test public void logDataWrittenIsEqualToLogDataRead() throws Exception { long address = 0L; LogData ld = getLogDataWithoutId(address); ld.setId(clientId1); client.write(ld); LogData ldPrime = client.read(address).get().getAddresses().get(address); assertThat(ld).isEqualTo(ldPrime); assertThat(ldPrime.getMetadataMap().get(IMetadata.LogUnitMetadataType.CLIENT_ID)) .isEqualTo(clientId1); assertThat(ldPrime.getMetadataMap().get(IMetadata.LogUnitMetadataType.THREAD_ID)) .isEqualTo(Thread.currentThread().getId()); }
@Test public void flushLogunitCache() throws Exception { LogUnitServer server2 = new LogUnitServer(serverContext); serverRouter.reset(); serverRouter.addServer(server2); assertThat(server2.getDataCache().asMap().size()).isEqualTo(0); byte[] testString = "hello world".getBytes(); client.write(0, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get(); assertThat(server2.getDataCache().asMap().size()).isEqualTo(1); client.flushCache().get(); assertThat(server2.getDataCache().asMap().size()).isEqualTo(0); LogData r = client.read(0).get().getAddresses().get(0L); assertThat(server2.getDataCache().asMap().size()).isEqualTo(1); }
@Test public void canReadWriteRanked() throws Exception { byte[] testString = "hello world".getBytes(); client.write(0, Collections.<UUID>emptySet(), new IMetadata.DataRank(1), testString, Collections.emptyMap()).get(); LogData r = client.read(0).get().getAddresses().get(0L); assertThat(r.getType()) .isEqualTo(DataType.DATA); assertThat(r.getPayload(new CorfuRuntime())) .isEqualTo(testString); byte[] testString2 = "hello world 2".getBytes(); client.write(0, Collections.<UUID>emptySet(), new IMetadata.DataRank(2), testString2, Collections.emptyMap()).get(); r = client.read(0).get().getAddresses().get(0L); assertThat(r.getType()) .isEqualTo(DataType.DATA); assertThat(r.getPayload(new CorfuRuntime())) .isEqualTo(testString2); }
@Test public void multiReadTest() throws Exception { byte[] payload = "payload".getBytes(); CorfuRuntimeParameters p = CorfuRuntimeParameters.builder().build(); final int numBatches = 3; for (long x = 0; x < numBatches * p.getBulkReadSize(); x++) { client.write(x, Collections.emptySet(), null, payload, Collections.emptyMap()).get(); } // Read half a batch List<Long> halfBatch = new ArrayList<>(); final int half = p.getBulkReadSize() / 2; for (long x = 0; x < half; x++) { halfBatch.add(x); } ReadResponse resp = client.read(halfBatch).get(); assertThat(resp.getAddresses().size()).isEqualTo(half); // Read two batches List<Long> twoBatchAddresses = new ArrayList<>(); final int twoBatches = p.getBulkReadSize() * 2; for (long x = 0; x < twoBatches; x++) { twoBatchAddresses.add(x); } resp = client.read(twoBatchAddresses).get(); assertThat(resp.getAddresses().size()).isEqualTo(twoBatches); }
@Test public void readingTrimmedAddress() throws Exception { byte[] testString = "hello world".getBytes(); final long address0 = 0; final long address1 = 1; client.write(address0, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get(); client.write(address1, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get(); LogData r = client.read(address0).get().getAddresses().get(0L); assertThat(r.getType()) .isEqualTo(DataType.DATA); r = client.read(address1).get().getAddresses().get(1L); assertThat(r.getType()) .isEqualTo(DataType.DATA); client.prefixTrim(new Token(0L, address0)); client.compact(); // For logunit cach flush LogUnitServer server2 = new LogUnitServer(serverContext); serverRouter.reset(); serverRouter.addServer(server2); LogData trimmedAddress = client.read(address0).get().getAddresses().get(0L); assertThat(trimmedAddress.isTrimmed()).isTrue(); assertThat(trimmedAddress.getGlobalAddress()).isEqualTo(address0); }
@Test public void cannotOutrank() throws ExecutionException, InterruptedException { byte[] testString = "hello world".getBytes(); client.write(0, Collections.<UUID>emptySet(), new IMetadata.DataRank(2), testString, Collections.emptyMap()).get(); LogData r = client.read(0).get().getAddresses().get(0L); assertThat(r.getType()) .isEqualTo(DataType.DATA); assertThat(r.getPayload(new CorfuRuntime())) .isEqualTo(testString); byte[] testString2 = "hello world 2".getBytes(); try { client.write(0, Collections.<UUID>emptySet(), new IMetadata.DataRank(1), testString2, Collections.emptyMap()).get(); fail(); } catch (ExecutionException e) { // expected assertEquals(DataOutrankedException.class, e.getCause().getClass()); } r = client.read(0).get().getAddresses().get(0L); assertThat(r.getType()) .isEqualTo(DataType.DATA); assertThat(r.getPayload(new CorfuRuntime())) .isEqualTo(testString); }
@Test public void backpointersCanBeWrittenAndRead() throws Exception { final long ADDRESS_0 = 1337L; final long ADDRESS_1 = 1338L; byte[] testString = "hello world".getBytes(); client.write(0, Collections.<UUID>emptySet(), null, testString, ImmutableMap.<UUID, Long>builder() .put(CorfuRuntime.getStreamID("hello"), ADDRESS_0) .put(CorfuRuntime.getStreamID("hello2"), ADDRESS_1) .build()).get(); LogData r = client.read(0).get().getAddresses().get(0L); assertThat(r.getBackpointerMap()) .containsEntry(CorfuRuntime.getStreamID("hello"), ADDRESS_0); assertThat(r.getBackpointerMap()) .containsEntry(CorfuRuntime.getStreamID("hello2"), ADDRESS_1); }
@Test public void holeFillCannotOverwrite() throws Exception { byte[] testString = "hello world".getBytes(); client.write(0, Collections.<UUID>emptySet(), null, testString, Collections.emptyMap()).get(); LogData r = client.read(0).get().getAddresses().get(0L); assertThat(r.getType()) .isEqualTo(DataType.DATA); Condition<Throwable> conditionOverwrite = new Condition<>(e -> { if (e.getCause().getClass().equals(OverwriteException.class)) { OverwriteException oe = (OverwriteException) e.getCause(); return oe.getOverWriteCause().getId() == OverwriteCause.DIFF_DATA.getId(); } else { return false; } }, "Expected overwrite cause to be DIFF_DATA"); Token token = new Token(0, 0); assertThatThrownBy(() -> client.fillHole(token).get()) .isInstanceOf(ExecutionException.class) .hasCauseInstanceOf(OverwriteException.class) .has(conditionOverwrite); }
@Test public void valueCanBeAdopted() throws ExecutionException, InterruptedException { byte[] testString = "hello world".getBytes(); client.write(0, Collections.<UUID>emptySet(), new IMetadata.DataRank(1), testString, Collections.emptyMap()).get(); LogData r = client.read(0).get().getAddresses().get(0L); assertThat(r.getType()) .isEqualTo(DataType.DATA); assertThat(r.getPayload(new CorfuRuntime())) .isEqualTo(testString); try { ILogData data = createEmptyData(0, DataType.RANK_ONLY, new IMetadata.DataRank(2)).getSerialized(); client.write(data).get(); fail(); } catch (Exception e) { // expected assertEquals(ValueAdoptedException.class, e.getCause().getClass()); ValueAdoptedException ex = (ValueAdoptedException)e.getCause(); ReadResponse read = ex.getReadResponse(); LogData log = read.getAddresses().get(0l); assertThat(log.getType()).isEqualTo(DataType.DATA); assertThat(log.getPayload(new CorfuRuntime())).isEqualTo(testString);; } r = client.read(0).get().getAddresses().get(0L); assertThat(r.getType()).isEqualTo(DataType.DATA); assertThat(r.getPayload(new CorfuRuntime())).isEqualTo(testString); }
/** * Ensure that same LogData payload written by two different client * are not equals * @throws Exception */ @Test public void logDataWrittenWithOtherClientIdIsNotEqual() throws Exception { long address = 0L; LogData ldOne = getLogDataWithoutId(address); LogData ldTwo = getLogDataWithoutId(address); ldOne.setId(clientId1); ldTwo.setId(clientId2); client.write(ldOne); LogData ldRead = client.read(address).get().getAddresses().get(address); assertThat(ldRead).isEqualTo(ldOne); assertThat(ldRead).isNotEqualTo(ldTwo); } }
/** * Ensure that same LogData payload written by two different thread * are not equals * @throws Exception */ @Test public void logDataWrittenFromOtherThreadIsNotEqual() throws Exception { long address = 0L; LogData ldThisThread = getLogDataWithoutId(address); ldThisThread.setId(clientId1); LogData ldOtherThread = getLogDataWithoutId(address); // Set clientId from another thread t1(() -> ldOtherThread.setId(clientId1)); client.write(ldOtherThread); LogData ldPrime = client.read(address).get().getAddresses().get(address); assertThat(ldThisThread).isNotEqualTo(ldPrime); assertThat(ldOtherThread).isEqualTo(ldPrime); }
/** * {@inheritDoc} */ @Override public void write(RuntimeLayout runtimeLayout, ILogData data) throws OverwriteException { final long globalAddress = data.getGlobalAddress(); int numUnits = runtimeLayout.getLayout().getSegmentLength(globalAddress); // To reduce the overhead of serialization, we serialize only the // first time we write, saving when we go down the chain. try (ILogData.SerializationHandle sh = data.getSerializedForm()) { log.trace("Write[{}]: chain head {}/{}", globalAddress, 1, numUnits); // In chain replication, we start at the chain head. try { CFUtils.getUninterruptibly( runtimeLayout.getLogUnitClient(globalAddress, 0) .write(sh.getSerialized()), OverwriteException.class); propagate(runtimeLayout, globalAddress, sh.getSerialized()); } catch (OverwriteException oe) { // Some other wrote here (usually due to hole fill) // We need to invoke the recovery protocol, in case // the write wasn't driven to completion. recover(runtimeLayout, globalAddress); throw oe; } } }
/** Check to see that a read correctly * completes a failed write from another client. */ @Test public void failedWriteCanBeRead() { setupNodes(); //begin tests final CorfuRuntime r = getDefaultRuntime(); final IReplicationProtocol rp = getProtocol(); final RuntimeLayout runtimeLayout = r.getLayoutView().getRuntimeLayout(); LogData incompleteWrite = getLogData(0, "incomplete".getBytes()); // Write the incomplete write to the head of the chain runtimeLayout.getLogUnitClient(SERVERS.ENDPOINT_0).write(incompleteWrite); // At this point, a read // reflect the -other- clients value ILogData readResult = rp.read(runtimeLayout, 0); assertThat(readResult.getPayload(r)) .isEqualTo("incomplete".getBytes()); }
/** Check to see that a writer correctly * completes a failed write from another client. */ @Test public void failedWriteIsPropagated() throws Exception { setupNodes(); //begin tests final CorfuRuntime r = getDefaultRuntime(); final IReplicationProtocol rp = getProtocol(); final RuntimeLayout runtimeLayout = r.getLayoutView().getRuntimeLayout(); LogData failedWrite = getLogData(0, "failed".getBytes()); LogData incompleteWrite = getLogData(0, "incomplete".getBytes()); // Write the incomplete write to the head of the chain runtimeLayout.getLogUnitClient(SERVERS.ENDPOINT_0).write(incompleteWrite); // Attempt to write using the replication protocol. // Should result in an overwrite exception assertThatThrownBy(() -> rp.write(runtimeLayout, failedWrite)) .isInstanceOf(OverwriteException.class); // At this point, a direct read of the tail should // reflect the -other- clients value ILogData readResult = runtimeLayout.getLogUnitClient(SERVERS.ENDPOINT_0) .read(0).get().getAddresses().get(0L); assertThat(readResult.getPayload(r)) .isEqualTo("incomplete".getBytes()); }
@Test public void canReadRankOnlyEntries() throws Exception { populateMaps(1, getDefaultRuntime(), CorfuTable.class, true, 2); LogUnitClient luc = getDefaultRuntime().getLayoutView().getRuntimeLayout() .getLogUnitClient(getDefaultConfigurationString()); SequencerClient seq = getDefaultRuntime().getLayoutView().getRuntimeLayout() .getSequencerClient(getDefaultConfigurationString()); long address = seq.nextToken(Collections.emptyList(),1).get().getSequence(); ILogData data = Helpers.createEmptyData(address, DataType.RANK_ONLY, new IMetadata.DataRank(2)) .getSerialized(); luc.write(data).get(); populateMaps(1, getDefaultRuntime(), CorfuTable.class, false, 1); address = seq.nextToken(Collections.emptyList(),1).get().getSequence(); data = Helpers.createEmptyData(address, DataType.RANK_ONLY, new IMetadata.DataRank(2)) .getSerialized(); luc.write(data).get(); populateMaps(1, getDefaultRuntime(), CorfuTable.class, false, 1); CorfuRuntime rt2 = Helpers.createNewRuntimeWithFastLoader(getDefaultConfigurationString()); assertThatMapsAreBuilt(rt2); assertThatObjectCacheIsTheSameSize(getDefaultRuntime(), rt2); }