private static void readSegmentFile(File file, long start, long end, boolean isOpen, Consumer<LogEntryProto> entryConsumer) throws IOException { try (LogInputStream in = new LogInputStream(file, start, end, isOpen)) { LogEntryProto next; LogEntryProto prev = null; while ((next = in.nextEntry()) != null) { if (prev != null) { Preconditions.assertTrue(next.getIndex() == prev.getIndex() + 1, "gap between entry %s and entry %s", prev, next); } if (entryConsumer != null) { entryConsumer.accept(next); } prev = next; } } }
@Override public String toString() { return getName(); }
while (end < maxIndexToScan) { long index; lastPos = in.getPosition(); try { if (hitError) { LogEntryProto entry = in.nextEntry(); index = entry != null ? entry.getIndex() : INVALID_LOG_INDEX; LOG.warn("After resync, position is " + in.getPosition()); } else { index = in.scanNextEntry();
/** * @param file File being scanned and validated. * @param maxTxIdToScan Maximum Tx ID to try to scan. * The scan returns after reading this or a higher * ID. The file portion beyond this ID is * potentially being updated. * @return Result of the validation * @throws IOException */ static LogValidation scanEditLog(File file, long maxTxIdToScan) throws IOException { LogInputStream in; try { in = new LogInputStream(file, INVALID_LOG_INDEX, INVALID_LOG_INDEX, false); // read the header, initialize the inputstream in.init(); } catch (EOFException e) { LOG.warn("Log file " + file + " has no valid header", e); return new LogValidation(0, INVALID_LOG_INDEX, true); } try { return scanEditLog(in, maxTxIdToScan); } finally { IOUtils.cleanup(LOG, in); } }
case UNINIT: try { init(); } catch (Throwable e) { LOG.error("caught exception initializing " + this, e); return nextEntry(); case OPEN: entry = reader.readEntry(); if (entry != null) { long index = entry.getIndex(); if (!isOpen() && index >= endIndex) { if (skipAmt > 0) { LOG.debug("skipping {} bytes at the end of log '{}': reached" + " entry {} out of {}", skipAmt, getName(), index, endIndex); reader.skipFully(skipAmt);
public LogEntryProto nextEntry() throws IOException { if (state.isUnopened()) { try { init(); } catch (Throwable e) { LOG.error("caught exception initializing " + this, e); if (entry != null) { long index = entry.getIndex(); if (!isOpen() && index >= endIndex) { if (skipAmt > 0) { LOG.debug("skipping {} bytes at the end of log '{}': reached" + " entry {} out of {}", skipAmt, getName(), index, endIndex); reader.skipFully(skipAmt);
/** * @param file File being scanned and validated. * @param maxTxIdToScan Maximum Tx ID to try to scan. * The scan returns after reading this or a higher * ID. The file portion beyond this ID is * potentially being updated. * @return Result of the validation * @throws IOException */ static LogValidation scanEditLog(File file, long maxTxIdToScan) throws IOException { LogInputStream in; try { in = new LogInputStream(file, INVALID_LOG_INDEX, INVALID_LOG_INDEX, false); // read the header, initialize the inputstream in.init(); } catch (EOFException e) { LOG.warn("Log file " + file + " has no valid header", e); return new LogValidation(0, INVALID_LOG_INDEX, true); } try { return scanEditLog(in, maxTxIdToScan); } finally { IOUtils.cleanup(LOG, in); } }
private LogEntryProto[] readLog(File file, long startIndex, long endIndex, boolean isOpen) throws IOException { List<LogEntryProto> list = new ArrayList<>(); try (LogInputStream in = new LogInputStream(file, startIndex, endIndex, isOpen)) { LogEntryProto entry; while ((entry = in.nextEntry()) != null) { list.add(entry); } } return list.toArray(new LogEntryProto[list.size()]); }
while (end < maxIndexToScan) { long index; lastPos = in.getPosition(); try { if (hitError) { LogEntryProto entry = in.nextEntry(); index = entry != null ? entry.getIndex() : INVALID_LOG_INDEX; LOG.warn("After resync, position is " + in.getPosition()); } else { index = in.scanNextEntry();
@Override public String toString() { return getName(); }
private static int readSegmentFile(File file, long start, long end, boolean isOpen, Consumer<LogEntryProto> entryConsumer) throws IOException { int count = 0; try (LogInputStream in = new LogInputStream(file, start, end, isOpen)) { for(LogEntryProto prev = null, next; (next = in.nextEntry()) != null; prev = next) { if (prev != null) { Preconditions.assertTrue(next.getIndex() == prev.getIndex() + 1, "gap between entry %s and entry %s", prev, next); } if (entryConsumer != null) { entryConsumer.accept(next); } count++; } } return count; }
public LogInputStream(File log, long startIndex, long endIndex, boolean isOpen) { if (isOpen) { Preconditions.assertTrue(endIndex == INVALID_LOG_INDEX); } else { Preconditions.assertTrue(endIndex >= startIndex); } this.logFile = log; this.startIndex = startIndex; this.endIndex = endIndex; this.isOpen = isOpen; this.state = new OpenCloseState(getName()); }
private LogEntryProto[] readLog(File file, long startIndex, long endIndex, boolean isOpen) throws IOException { List<LogEntryProto> list = new ArrayList<>(); try (LogInputStream in = new LogInputStream(file, startIndex, endIndex, isOpen)) { LogEntryProto entry; while ((entry = in.nextEntry()) != null) { list.add(entry); } } return list.toArray(new LogEntryProto[list.size()]); }
try (LogInputStream in = new LogInputStream(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true)) { LogEntryProto entry; while ((entry = in.nextEntry()) != null) { list.add(entry);
try (LogInputStream in = new LogInputStream(openSegment, 0, RaftServerConstants.INVALID_LOG_INDEX, true)) { LogEntryProto entry; while ((entry = in.nextEntry()) != null) { list.add(entry);
new LogInputStream(file, 0, INVALID_LOG_INDEX, true)) { LogEntryProto entry = in.nextEntry(); Assert.assertNull(entry); try (LogInputStream in = new LogInputStream(file, 0, INVALID_LOG_INDEX, true)) { LogEntryProto entry = in.nextEntry(); Assert.assertArrayEquals(content, entry.getSmLogEntry().getData().toByteArray()); Assert.assertNull(in.nextEntry());
private synchronized long loadSnapshot(SingleFileSnapshotInfo snapshot) throws IOException { if (snapshot == null || !snapshot.getFile().getPath().toFile().exists()) { LOG.info("The snapshot file {} does not exist", snapshot == null ? null : snapshot.getFile()); return RaftServerConstants.INVALID_LOG_INDEX; } else { LOG.info("Loading snapshot with t:{}, i:{}, file:{}", snapshot.getTerm(), snapshot.getIndex(), snapshot.getFile().getPath()); final long endIndex = snapshot.getIndex(); try (LogInputStream in = new LogInputStream( snapshot.getFile().getPath().toFile(), 0, endIndex, false)) { LogEntryProto entry; while ((entry = in.nextEntry()) != null) { list.add(entry); updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); } } Preconditions.assertTrue( !list.isEmpty() && endIndex == list.get(list.size() - 1).getIndex(), "endIndex=%s, list=%s", endIndex, list); this.endIndexLastCkpt = endIndex; setLastAppliedTermIndex(snapshot.getTermIndex()); this.storage.loadLatestSnapshot(); return endIndex; } }
new LogInputStream(file, 0, INVALID_LOG_INDEX, true)) { LogEntryProto entry = in.nextEntry(); Assert.assertNull(entry); try (LogInputStream in = new LogInputStream(file, 0, INVALID_LOG_INDEX, true)) { LogEntryProto entry = in.nextEntry(); Assert.assertArrayEquals(content, entry.getStateMachineLogEntry().getLogData().toByteArray()); Assert.assertNull(in.nextEntry());
private synchronized long loadSnapshot(SingleFileSnapshotInfo snapshot) throws IOException { if (snapshot == null || !snapshot.getFile().getPath().toFile().exists()) { LOG.info("The snapshot file {} does not exist", snapshot == null ? null : snapshot.getFile()); return RaftServerConstants.INVALID_LOG_INDEX; } else { LOG.info("Loading snapshot with t:{}, i:{}, file:{}", snapshot.getTerm(), snapshot.getIndex(), snapshot.getFile().getPath()); final long endIndex = snapshot.getIndex(); try (LogInputStream in = new LogInputStream( snapshot.getFile().getPath().toFile(), 0, endIndex, false)) { LogEntryProto entry; while ((entry = in.nextEntry()) != null) { put(entry); updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); } } Preconditions.assertTrue( !indexMap.isEmpty() && endIndex == indexMap.lastKey(), "endIndex=%s, indexMap=%s", endIndex, indexMap); this.endIndexLastCkpt = endIndex; setLastAppliedTermIndex(snapshot.getTermIndex()); this.storage.loadLatestSnapshot(); return endIndex; } }