private CompletableFuture<Void> appendData(Collection<String> segmentNames, HashMap<String, ByteArrayOutputStream> segmentContents, HashMap<String, Long> lengths, StreamSegmentStore store) { return execute(createAppendDataRequests(segmentNames, segmentContents, lengths), store); }
try (val builder = createBuilder(++instanceId)) { val segmentStore = builder.createStreamSegmentService(); segmentNames = createSegments(segmentStore); log.info("Created Segments: {}.", String.join(", ", segmentNames)); transactionsBySegment = createTransactions(segmentNames, segmentStore); log.info("Created Transactions: {}.", transactionsBySegment.values().stream().flatMap(Collection::stream).collect(Collectors.joining(", "))); appendData(segmentsAndTransactions, segmentContents, lengths, segmentStore).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); log.info("Finished appending data."); checkSegmentStatus(lengths, startOffsets, false, false, segmentStore); log.info("Finished Phase 1"); try (val builder = createBuilder(++instanceId)) { val segmentStore = builder.createStreamSegmentService(); checkReads(segmentContents, segmentStore); log.info("Finished checking reads."); mergeTransactions(transactionsBySegment, lengths, segmentContents, segmentStore).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); log.info("Finished merging transactions."); checkSegmentStatus(lengths, startOffsets, false, false, segmentStore); log.info("Finished Phase 2."); try (val builder = createBuilder(++instanceId); val readOnlyBuilder = createReadOnlyBuilder(instanceId)) { val segmentStore = builder.createStreamSegmentService();
val segmentNames = createSegments(context.getActiveStore()); val segmentsAndTransactions = new ArrayList<String>(segmentNames); log.info("Created Segments: {}.", String.join(", ", segmentNames)); HashMap<String, Long> startOffsets = new HashMap<>(); HashMap<String, ByteArrayOutputStream> segmentContents = new HashMap<>(); val appends = createAppendDataRequests(segmentsAndTransactions, segmentContents, lengths, applyFencingMultiplier(ATTRIBUTE_UPDATES_PER_SEGMENT), applyFencingMultiplier(APPENDS_PER_SEGMENT)); val requests = appends.iterator(); int newInstanceFrequency = appends.size() / applyFencingMultiplier(MAX_INSTANCE_COUNT); log.info("Creating a new Segment Store instance every {} operations.", newInstanceFrequency); val operationCompletions = executeWithFencing(requests, newInstanceFrequency, context); checkReads(segmentContents, context.getActiveStore()); log.info("Finished checking reads."); try (val readOnlyBuilder = createReadOnlyBuilder(Integer.MAX_VALUE - 1)) { waitForSegmentsInStorage(segmentNames, context.getActiveStore(), readOnlyBuilder.createStreamSegmentService()) .get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); log.info("Finished waiting for segments in Storage."); deleteSegments(segmentNames, context.getActiveStore()).join(); log.info("Finished deleting segments."); checkSegmentStatus(lengths, startOffsets, true, true, context.getActiveStore());
private ArrayList<StoreRequest> createAppendDataRequests( Collection<String> segmentNames, HashMap<String, ByteArrayOutputStream> segmentContents, HashMap<String, Long> lengths, int attributeUpdatesPerSegment, int appendsPerSegment) { val result = new ArrayList<StoreRequest>(); val halfAttributeCount = attributeUpdatesPerSegment / 2; for (String segmentName : segmentNames) { if (isEmptySegment(segmentName)) { continue; } // Add half the attribute updates now. for (int i = 0; i < halfAttributeCount; i++) { result.add(store -> store.updateAttributes(segmentName, createAttributeUpdates(), TIMEOUT)); } // Add some appends. for (int i = 0; i < appendsPerSegment; i++) { byte[] appendData = getAppendData(segmentName, i); lengths.put(segmentName, lengths.getOrDefault(segmentName, 0L) + appendData.length); recordAppend(segmentName, appendData, segmentContents); result.add(store -> store.append(segmentName, appendData, createAttributeUpdates(), TIMEOUT)); } // Add the rest of the attribute updates. for (int i = 0; i < halfAttributeCount; i++) { result.add(store -> store.updateAttributes(segmentName, createAttributeUpdates(), TIMEOUT)); } } return result; }
private ServiceBuilder createBuilder(int instanceId) throws Exception { val builder = createBuilder(this.configBuilder, instanceId); try { builder.initialize(); } catch (Throwable ex) { builder.close(); throw ex; } return builder; }
private ArrayList<StoreRequest> createAppendDataRequests( Collection<String> segmentNames, HashMap<String, ByteArrayOutputStream> segmentContents, HashMap<String, Long> lengths) { return createAppendDataRequests(segmentNames, segmentContents, lengths, ATTRIBUTE_UPDATES_PER_SEGMENT, APPENDS_PER_SEGMENT); }
private void checkReads(HashMap<String, ByteArrayOutputStream> segmentContents, StreamSegmentStore store) { for (Map.Entry<String, ByteArrayOutputStream> e : segmentContents.entrySet()) { String segmentName = e.getKey(); byte[] expectedData = e.getValue().toByteArray(); long segmentLength = store.getStreamSegmentInfo(segmentName, TIMEOUT).join().getLength(); Assert.assertEquals("Unexpected Read Index length for segment " + segmentName, expectedData.length, segmentLength); AtomicLong expectedCurrentOffset = new AtomicLong(0); // We retry a number of times on StreamSegmentNotExists. It is possible that waitForSegmentsInStorage may have // returned successfully because it detected the Segment was complete there, but the internal callback to the // ReadIndex (completeMerge) may not yet have been executed. The ReadIndex has a mechanism to cope with this, // but it only retries once, after a fixed time interval, which is more than generous on any system. // However, on very slow systems, it is possible that that callback may take a significant amount of time to even // begin executing, hence the trying to read data that was merged from a Transaction may result in a spurious // StreamSegmentNotExistsException. // This is gracefully handled by retries in AppendProcessor and/or Client, but in this case, we simply have to // do the retries ourselves, hoping that the callback eventually executes. Retry.withExpBackoff(100, 2, 10, TIMEOUT.toMillis() / 5) .retryWhen(ex -> Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException) .run(() -> { checkSegmentReads(segmentName, expectedCurrentOffset, segmentLength, store, expectedData); return null; }); } }
private ServiceBuilder createReadOnlyBuilder(int instanceId) throws Exception { // Copy base config properties to a new object. val props = new Properties(); this.configBuilder.build().forEach(props::put); // Create a new config (so we don't alter the base one) and set the ReadOnlySegmentStore to true). val configBuilder = ServiceBuilderConfig.builder() .include(props) .include(ServiceConfig.builder() .with(ServiceConfig.READONLY_SEGMENT_STORE, true)); val builder = createBuilder(configBuilder, instanceId); builder.initialize(); return builder; }