private void startReadFile() throws IOException { currentFileInfo = files.get(fileIndex); File snapshotFile = currentFileInfo.getPath().toFile(); currentFileSize = snapshotFile.length(); final int bufLength = getSnapshotChunkLength(currentFileSize); currentBuf = new byte[bufLength]; currentOffset = 0; chunkIndex = 0; in = new FileInputStream(snapshotFile); }
public SingleFileSnapshotInfo findLatestSnapshot() throws IOException { SingleFileSnapshotInfo latest = null; try (DirectoryStream<Path> stream = Files.newDirectoryStream(smDir.toPath())) { for (Path path : stream) { Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString()); if (matcher.matches()) { final long endIndex = Long.parseLong(matcher.group(2)); if (latest == null || endIndex > latest.getIndex()) { final long term = Long.parseLong(matcher.group(1)); MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile()); final FileInfo fileInfo = new FileInfo(path, fileDigest); latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex); } } } } return latest; }
private FileChunkProto readFileChunk(FileInfo fileInfo, FileInputStream in, byte[] buf, int length, long offset, int chunkIndex) throws IOException { FileChunkProto.Builder builder = FileChunkProto.newBuilder() .setOffset(offset).setChunkIndex(chunkIndex); IOUtils.readFully(in, buf, 0, length); Path relativePath = server.getState().getStorage().getStorageDir() .relativizeToRoot(fileInfo.getPath()); builder.setFilename(relativePath.toString()); builder.setDone(offset + length == fileInfo.getFileSize()); builder.setFileDigest( ByteString.copyFrom(fileInfo.getFileDigest().getDigest())); builder.setData(ByteString.copyFrom(buf, 0, length)); return builder.build(); }
private FileChunkProto readFileChunk(FileInfo fileInfo, FileInputStream in, byte[] buf, int length, long offset, int chunkIndex) throws IOException { FileChunkProto.Builder builder = FileChunkProto.newBuilder() .setOffset(offset).setChunkIndex(chunkIndex); IOUtils.readFully(in, buf, 0, length); Path relativePath = server.getState().getStorage().getStorageDir() .relativizeToRoot(fileInfo.getPath()); builder.setFilename(relativePath.toString()); builder.setDone(offset + length == fileInfo.getFileSize()); builder.setFileDigest( ByteString.copyFrom(fileInfo.getFileDigest().getDigest())); builder.setData(ByteString.copyFrom(buf, 0, length)); return builder.build(); }
private void startReadFile() throws IOException { currentFileInfo = files.get(fileIndex); File snapshotFile = currentFileInfo.getPath().toFile(); currentFileSize = snapshotFile.length(); final int bufLength = getSnapshotChunkLength(currentFileSize); currentBuf = new byte[bufLength]; currentOffset = 0; chunkIndex = 0; in = new FileInputStream(snapshotFile); }
public SingleFileSnapshotInfo findLatestSnapshot() throws IOException { SingleFileSnapshotInfo latest = null; try (DirectoryStream<Path> stream = Files.newDirectoryStream(smDir.toPath())) { for (Path path : stream) { Matcher matcher = SNAPSHOT_REGEX.matcher(path.getFileName().toString()); if (matcher.matches()) { final long endIndex = Long.parseLong(matcher.group(2)); if (latest == null || endIndex > latest.getIndex()) { final long term = Long.parseLong(matcher.group(1)); MD5Hash fileDigest = MD5FileUtil.readStoredMd5ForFile(path.toFile()); final FileInfo fileInfo = new FileInfo(path, fileDigest); latest = new SingleFileSnapshotInfo(fileInfo, term, endIndex); } } } } return latest; }
private long load(SingleFileSnapshotInfo snapshot, boolean reload) throws IOException { if (snapshot == null) { LOG.warn("The snapshot info is null."); return RaftServerConstants.INVALID_LOG_INDEX; } final File snapshotFile = snapshot.getFile().getPath().toFile(); if (!snapshotFile.exists()) { LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot); return RaftServerConstants.INVALID_LOG_INDEX; } final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile); try(final AutoCloseableLock writeLock = writeLock(); final ObjectInputStream in = new ObjectInputStream( new BufferedInputStream(new FileInputStream(snapshotFile)))) { if (reload) { reset(); } setLastAppliedTermIndex(last); variables.putAll((Map<String, Double>) in.readObject()); } catch (ClassNotFoundException e) { throw new IllegalStateException(e); } return last.getIndex(); }
private long load(SingleFileSnapshotInfo snapshot, boolean reload) throws IOException { if (snapshot == null) { LOG.warn("The snapshot info is null."); return RaftServerConstants.INVALID_LOG_INDEX; } final File snapshotFile = snapshot.getFile().getPath().toFile(); if (!snapshotFile.exists()) { LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot); return RaftServerConstants.INVALID_LOG_INDEX; } final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile); try(final AutoCloseableLock writeLock = writeLock(); final ObjectInputStream in = new ObjectInputStream( new BufferedInputStream(new FileInputStream(snapshotFile)))) { if (reload) { reset(); } setLastAppliedTermIndex(last); variables.putAll(JavaUtils.cast(in.readObject())); } catch (ClassNotFoundException e) { throw new IllegalStateException(e); } return last.getIndex(); }
private long load(SingleFileSnapshotInfo snapshot, boolean reload) throws IOException { if (snapshot == null) { LOG.warn("The snapshot info is null."); return RaftServerConstants.INVALID_LOG_INDEX; } final File snapshotFile = snapshot.getFile().getPath().toFile(); if (!snapshotFile.exists()) { LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot); return RaftServerConstants.INVALID_LOG_INDEX; } final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile); try(final AutoCloseableLock writeLock = writeLock(); final ObjectInputStream in = new ObjectInputStream( new BufferedInputStream(new FileInputStream(snapshotFile)))) { if (reload) { reset(); } setLastAppliedTermIndex(last); this.length = in.readLong(); this.state = (State) in.readObject(); } catch (ClassNotFoundException e) { throw new IllegalStateException(e); } return last.getIndex(); }
private synchronized long loadSnapshot(SingleFileSnapshotInfo snapshot) throws IOException { if (snapshot == null || !snapshot.getFile().getPath().toFile().exists()) { LOG.info("The snapshot file {} does not exist", snapshot == null ? null : snapshot.getFile()); return RaftServerConstants.INVALID_LOG_INDEX; } else { LOG.info("Loading snapshot with t:{}, i:{}, file:{}", snapshot.getTerm(), snapshot.getIndex(), snapshot.getFile().getPath()); final long endIndex = snapshot.getIndex(); try (LogInputStream in = new LogInputStream( snapshot.getFile().getPath().toFile(), 0, endIndex, false)) { LogEntryProto entry; while ((entry = in.nextEntry()) != null) { list.add(entry); updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); } } Preconditions.assertTrue( !list.isEmpty() && endIndex == list.get(list.size() - 1).getIndex(), "endIndex=%s, list=%s", endIndex, list); this.endIndexLastCkpt = endIndex; setLastAppliedTermIndex(snapshot.getTermIndex()); this.storage.loadLatestSnapshot(); return endIndex; } }
private synchronized long loadSnapshot(SingleFileSnapshotInfo snapshot) throws IOException { if (snapshot == null || !snapshot.getFile().getPath().toFile().exists()) { LOG.info("The snapshot file {} does not exist", snapshot == null ? null : snapshot.getFile()); return RaftServerConstants.INVALID_LOG_INDEX; } else { LOG.info("Loading snapshot with t:{}, i:{}, file:{}", snapshot.getTerm(), snapshot.getIndex(), snapshot.getFile().getPath()); final long endIndex = snapshot.getIndex(); try (LogInputStream in = new LogInputStream( snapshot.getFile().getPath().toFile(), 0, endIndex, false)) { LogEntryProto entry; while ((entry = in.nextEntry()) != null) { put(entry); updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); } } Preconditions.assertTrue( !indexMap.isEmpty() && endIndex == indexMap.lastKey(), "endIndex=%s, indexMap=%s", endIndex, indexMap); this.endIndexLastCkpt = endIndex; setLastAppliedTermIndex(snapshot.getTermIndex()); this.storage.loadLatestSnapshot(); return endIndex; } }