private void read(@NotNull ChronicleQueue aChronicle, int expected) { final ExcerptTailer myTailer = aChronicle.createTailer(); final int myLast = HOURLY.toCycle(myTailer.toEnd().index()); final int myFirst = HOURLY.toCycle(myTailer.toStart().index()); int myCycle = myFirst - 1; long myIndex = HOURLY.toIndex(myCycle, 0); int count = 0; while (myCycle <= myLast) { // System.out.println(Long.toHexString(myIndex)); if (myTailer.moveToIndex(myIndex)) { while (myTailer.readDocument(read())) { count++; } } myIndex = HOURLY.toIndex(++myCycle, 0); } Assert.assertEquals(expected, count); }
throw new IllegalArgumentException("You must pass the queue written to, not the queue read"); @NotNull ExcerptTailer tailer = queue.createTailer() .direction(BACKWARD) .toEnd(); try (DocumentContext context = tailer.readingDocument()) { if (!context.isPresent()) { toStart();
void runLoop() { try { JDBCResult result = out.acquireAppender() .methodWriterBuilder(JDBCResult.class) .recordHistory(true) .get(); JDBCComponent js = new JDBCComponent(connectionSupplier, result); MethodReader reader = in.createTailer().afterLastWritten(out).methodReader(js); Pauser pauser = Pauser.millis(1, 10); while (!closed) { if (reader.readOne()) pauser.reset(); else pauser.pause(); } } catch (Throwable t) { LOGGER.warn("Run loop exited", t); } }
private String extraInfo(@NotNull ExcerptTailer tailer, @NotNull VanillaMessageHistory messageHistory) { return String.format( ". That sourceIndex was determined fom the last entry written to queue %s " + "(message index %s, message history %s). If source queue is replicated then " + "sourceIndex may not have been replicated yet", tailer.queue().fileAbsolutePath(), Long.toHexString(tailer.index()), WireType.TEXT.asString(messageHistory)); }
private void moveToSpecifiedPosition(final ChronicleQueue ic, final ExcerptTailer tailer, final boolean isFirstIteration) { if (isSet(startIndex) && isFirstIteration) { if (startIndex < ic.firstIndex()) { throw new IllegalArgumentException(String.format("startIndex %d is less than first index %d", startIndex, ic.firstIndex())); } messageSink.accept("Waiting for startIndex " + startIndex); for (; ; ) { if (tailer.moveToIndex(startIndex)) break; Jvm.pause(100); } } if (isSet(maxHistoryRecords) && isFirstIteration) { tailer.toEnd(); tailer.moveToIndex(Math.max(ic.firstIndex(), tailer.index() - maxHistoryRecords)); } else if (tailInputSource && isFirstIteration) { tailer.toEnd(); } }
private long action(@NotNull final ExcerptTailer tailer1, @NotNull final RollCycle rollCycle) { // tailer1.direction(direction); long readvalue = 0; long seqNumRead = 0; try (final DocumentContext dc = tailer1.readingDocument()) { readvalue = dc.wire().read("value").int64(); seqNumRead = dc.index(); } final long nextSeq = rollCycle.toSequenceNumber(tailer1.index()); /*System.out.println("Return-value=" + readvalue + ", seq=" + rollCycle.toSequenceNumber(seqNumRead) + ", next-seq=" + nextSeq + "(" + Long.toHexString(nextSeq) + "x0)" + ",direction=" + tailer1.direction());*/ return readvalue; }
tailer.toStart(); try (DocumentContext documentContext = tailer.readingDocument()) { long currentOffset = documentContext.index(); System.out.println("Current offset: " + currentOffset);
.direction(TailerDirection.FORWARD) .toStart(); try (DocumentContext context = forwardTailer.readingDocument()) { assertTrue(context.isPresent()); try (DocumentContext context = forwardTailer.readingDocument()) { assertFalse(context.isPresent()); .direction(TailerDirection.BACKWARD) .toEnd(); try (DocumentContext context = backwardTailer.readingDocument()) { assertTrue(context.isPresent());
private static long countEntries(final ChronicleQueue queue) { final ExcerptTailer tailer = queue.createTailer(); tailer.toStart().direction(TailerDirection.FORWARD); long entryCount = 0L; while (true) { try (final DocumentContext ctx = tailer.readingDocument()) { if (!ctx.isPresent()) { break; } entryCount++; } } return entryCount; }
@Test public void shouldHandleCycleRollWhenInReadOnlyMode() { final MutableTimeProvider timeProvider = new MutableTimeProvider(); final ChronicleQueue queue = build(createQueue(dataDirectory, RollCycles.MINUTELY, 0, "cycleRoll", false). timeProvider(timeProvider)); final StringEvents events = queue.acquireAppender().methodWriterBuilder(StringEvents.class).build(); timeProvider.setTime(System.currentTimeMillis()); events.onEvent("firstEvent"); timeProvider.addTime(2, TimeUnit.MINUTES); events.onEvent("secondEvent"); final ChronicleQueue readerQueue = build(createQueue(dataDirectory, RollCycles.MINUTELY, 0, "cycleRoll", true). timeProvider(timeProvider)); final ExcerptTailer tailer = readerQueue.createTailer(); tailer.toStart(); try (final DocumentContext context = tailer.readingDocument()) { assertThat(context.isPresent(), is(true)); } tailer.toEnd(); try (final DocumentContext context = tailer.readingDocument()) { assertThat(context.isPresent(), is(false)); } }
@Test public void testAppendAndRead() { try (final ChronicleQueue queue = builder(getTmpDir(), this.wireType) .build()) { final ExcerptAppender appender = queue.acquireAppender(); final int cycle = appender.cycle(); for (int i = 0; i < 10; i++) { final int n = i; appender.writeDocument(w -> w.write(TestKey.test).int32(n)); assertEquals(n, queue.rollCycle().toSequenceNumber(appender.lastIndexAppended())); } final ExcerptTailer tailer = queue.createTailer(); // Sequential read for (int i = 0; i < 10; i++) { final int n = i; assertTrue(tailer.readDocument(r -> assertEquals(n, r.read(TestKey.test).int32()))); assertEquals(n + 1, queue.rollCycle().toSequenceNumber(tailer.index())); } // Random read for (int i = 0; i < 10; i++) { final int n = i; assertTrue("n: " + n, tailer.moveToIndex(queue.rollCycle().toIndex(cycle, n))); assertTrue("n: " + n, tailer.readDocument(r -> assertEquals(n, r.read(TestKey.test).int32()))); assertEquals(n + 1, queue.rollCycle().toSequenceNumber(tailer.index())); } } }
@Test public void testRandomMove() throws Exception { final Map<Long, String> messageByIndex = new HashMap<>(); try (ChronicleQueue queue = SingleChronicleQueueBuilder. binary(tmpFolder.newFolder()).build()) { // create a queue and add some excerpts final ExcerptAppender appender = queue.acquireAppender(); for (int i = 0; i < 10; i++) { final String message = "msg" + i; appender.writeDocument(w -> w.write("message").object(message)); final long appendIndex = appender.lastIndexAppended(); messageByIndex.put(appendIndex, message); } final Random random = new Random(1510298038000L); final List<Long> indices = new ArrayList<>(messageByIndex.keySet()); final ExcerptTailer tailer = queue.createTailer(); final AtomicReference<String> capturedMessage = new AtomicReference<>(); for (int i = 0; i < 100; i++) { final long randomIndex = indices.get(random.nextInt(messageByIndex.keySet().size())); tailer.moveToIndex(randomIndex); tailer.readDocument(w -> capturedMessage.set((String) w.read("message").object())); assertEquals(messageByIndex.get(randomIndex), capturedMessage.get()); tailer.readDocument(w -> w.read("message").object()); } } }
private long toIndexFromEnd(long countFromEnd) { long fromSequenceNumber = -1; if (countFromEnd > 0) { @Nullable final SingleChronicleQueue chronicleQueue = (SingleChronicleQueue) queueView.underlying(); @NotNull final ExcerptTailer excerptTailer = chronicleQueue.createTailer().direction(BACKWARD).toEnd(); fromSequenceNumber = excerptTailer.index(); if (fromSequenceNumber == 0) return 0; for (int i = 0; i < countFromEnd; i++) { try (DocumentContext documentContext = excerptTailer.readingDocument()) { if (!documentContext.isPresent()) break; fromSequenceNumber = documentContext.index(); } } } return fromSequenceNumber; }
final ExcerptTailer tailer = queue.createTailer().toEnd(); try (final DocumentContext readCtx = tailer.readingDocument()) { assertThat(readCtx.isPresent(), is(false)); assertThat(tailer.state(), is(TailerState.END_OF_CYCLE)); tailer.direction(TailerDirection.BACKWARD); tailer.toEnd(); assertThat(tailer.readingDocument().isPresent(), is(true));
writer.append("# firstIndex: ").append(Long.toHexString(firstIndex)).append("\n"); ExcerptTailer tailer = createTailer(); if (!tailer.moveToIndex(fromIndex)) { if (firstIndex > fromIndex) { tailer.toStart(); } else { return; TextWire text = new TextWire(bytes); while (true) { try (DocumentContext dc = tailer.readingDocument()) { if (!dc.isPresent()) { writer.append("# no more messages at ").append(Long.toHexString(dc.index())).append("\n");
@Override public DocumentContext apply(final ExcerptTailer excerptTailer) { final DocumentContext documentContext = excerptTailer.readingDocument(); if (!documentContext.isPresent()) { invocationCount++; if (invocationCount >= maxPollsReturningEmptyDocument) { throw new ArithmeticException("For testing purposes"); } } return documentContext; } }
@Test public void testAppendedBeforeToEnd() { File dir = getTmpDir(); try (ChronicleQueue chronicle = builder(dir, this.wireType) .rollCycle(RollCycles.TEST_SECONDLY) .build(); ChronicleQueue chronicle2 = builder(dir, this.wireType) .rollCycle(RollCycles.TEST_SECONDLY) .build()) { ExcerptTailer tailer = chronicle.createTailer(); ExcerptAppender append = chronicle2.acquireAppender(); append.writeDocument(w -> w.write(() -> "test").text("text")); while (tailer.state() == TailerState.UNINITIALISED) tailer.toEnd(); try (DocumentContext dc = tailer.readingDocument()) { assertFalse(tailer.index() + " " + tailer.state(), dc.isPresent()); } append.writeDocument(w -> w.write(() -> "test").text("text2")); try (DocumentContext dc = tailer.readingDocument()) { assertTrue(dc.isPresent()); assertEquals("text2", dc.wire().read("test").text()); } } }
private static long findCycleLinearSearch(@NotNull NavigableSet<Long> cycles, Wire key, @NotNull Comparator<Wire> c, @NotNull ExcerptTailer tailer, @NotNull final ChronicleQueue queue) { final Iterator<Long> iterator = cycles.iterator(); if (!iterator.hasNext()) return -1; final RollCycle rollCycle = queue.rollCycle(); long prevIndex = iterator.next(); while (iterator.hasNext()) { final Long current = iterator.next(); final boolean b = tailer.moveToIndex(rollCycle.toIndex((int) (long) current, 0)); if (!b) return prevIndex; try (final DocumentContext dc = tailer.readingDocument()) { final int compare = c.compare(dc.wire(), key); if (compare == 0) return current; else if (compare > 0) return prevIndex; prevIndex = current; } } return prevIndex; }