public DBBuilder() { storeConfig = new StoreConfig(); }
public void init() { future = this.executorService.scheduleWithFixedDelay( new Runnable() { @Override public void run() { if (!running.compareAndSet(false, true)) { return; } try { compact(); } catch (Throwable t) { LOGGER.error("DataCompactor compactor error:" + t.getMessage(), t); } finally { running.set(false); } } }, storeConfig.getDataBlockCompactCheckInterval(), storeConfig.getDataBlockCompactCheckInterval(), TimeUnit.MILLISECONDS); }
public DataBlock(String fileName, StoreConfig storeConfig) throws IOException { this.fileName = fileName; this.fileId = Long.parseLong(fileName.substring(0, fileName.lastIndexOf("."))); this.file = new File(storeConfig.getDataPath(), fileName); this.storeConfig = storeConfig; this.fileSize = storeConfig.getDataBlockFileSize(); this.fileHeader = new DataBlockFileHeader(); this.flushDataService = new FlushDataService(); init(); }
public DB<K, V> create() { if (serializer == null) { this.serializer = new JsonStoreSerializer(); } if (storeConfig.getDbPath() == null) { storeConfig.setDbPath(new File(Constants.USER_HOME)); } storeConfig.setDataPath(new File(storeConfig.getDbPath(), "data")); storeConfig.setLogPath(new File(storeConfig.getDbPath(), "logs")); storeConfig.setIndexPath(new File(storeConfig.getDbPath(), "index")); return new DBImpl<K, V>(serializer, storeConfig); }
/** * 删除多余的快照数目 */ private void deleteOverSnapshot() throws IOException { String[] indexFiles = getIndexFiles(); if (indexFiles == null || indexFiles.length == 0) { return; } if (storeConfig.getMaxIndexSnapshotSize() > 1 && indexFiles.length > storeConfig.getMaxIndexSnapshotSize()) { for (int i = 0; i < indexFiles.length - storeConfig.getMaxIndexSnapshotSize(); i++) { FileUtils.delete(new File(storeConfig.getIndexPath(), indexFiles[i])); LOGGER.info("delete index snapshot [" + indexFiles[i] + "] succeed"); } } } }
if (length > storeConfig.getMaxxLogEntryLength()) { throw new DBException("Value size can not great than " + storeConfig.getMaxxLogEntryLength()); if (fileLength + length + ENTRY_HEAD_LENGTH > storeConfig.getTxLogFileSize()) { throw new CapacityNotEnoughException(); if (syncTimerTask == null || syncTimerTask.isDone()) { syncTimerTask = new FutureTimerTask("ltsdb-dblog-sync-timertask", syncCallable); syncTimer.schedule(syncTimerTask, storeConfig.getDbLogFlushInterval());
@Override public void run() { DataBlock.LOGGER.info(this.getServiceName() + " service started"); while (!this.isStopped()) { try { if (storeConfig.isEnableFlushDataInterval()) { Thread.sleep(storeConfig.getFlushDataInterval()); } else { waitForRunning(storeConfig.getFlushDataInterval()); } // 刷到磁盘 flushDisk(); } catch (Exception e) { DataBlock.LOGGER.error(this.getServiceName() + " error:" + e.getMessage(), e); } } } }
private String[] getIndexFiles() throws IOException { String[] indexFiles = storeConfig.getIndexPath().list(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.endsWith(".snapshot"); } }); if (indexFiles == null) { throw new IOException("can't list file in " + storeConfig.getIndexPath()); } if (indexFiles.length == 0) { return null; } Arrays.sort(indexFiles, new Comparator<String>() { @Override public int compare(String left, String right) { return left.compareTo(right); } }); return indexFiles; }
@Override public void run() { try { // 检查一下当改变的量达到一定量时要snapshot if (currentChangeNum.get() - lastSnapshotChangeNum.get() > storeConfig.getIndexSnapshotThreshold()) { indexSnapshot.snapshot(); } } catch (Throwable t) { LOGGER.error("SNAPSHOT Error", t); } } }, 3, 2, TimeUnit.SECONDS);
public void init() throws DBException { try { FileUtils.createDirIfNotExist(storeConfig.getDbPath()); storeTxLogEngine.init(); dataBlockEngine.init(); indexSnapshot.init(); } catch (Exception e) { throw new DBException("DB init error:" + e.getMessage(), e); } }
public DataBlockEngine(StoreSerializer serializer, StoreConfig storeConfig) { this.serializer = serializer; this.storeConfig = storeConfig; this.dataPath = storeConfig.getDataPath(); }
public void init() throws IOException { // 1. load from disk loadFromDisk(); // 2. replay txLog replayTxLog(); future = this.executorService.scheduleWithFixedDelay(new Runnable() { @Override public void run() { try { snapshot(); } catch (Throwable t) { LOGGER.error("MemIndexSnapshot snapshot error:" + t.getMessage(), t); } } }, storeConfig.getIndexSnapshotInterval(), storeConfig.getIndexSnapshotInterval(), TimeUnit.MILLISECONDS); }
public DB<K, V> create() { if (serializer == null) { this.serializer = new JsonStoreSerializer(); } if (storeConfig.getDbPath() == null) { storeConfig.setDbPath(new File(Constants.USER_HOME)); } storeConfig.setDataPath(new File(storeConfig.getDbPath(), "data")); storeConfig.setLogPath(new File(storeConfig.getDbPath(), "logs")); storeConfig.setIndexPath(new File(storeConfig.getDbPath(), "index")); return new DBImpl<K, V>(serializer, storeConfig); }
if (length > storeConfig.getMaxxLogEntryLength()) { throw new DBException("Value size can not great than " + storeConfig.getMaxxLogEntryLength()); if (fileLength + length + ENTRY_HEAD_LENGTH > storeConfig.getTxLogFileSize()) { throw new CapacityNotEnoughException(); if (syncTimerTask == null || syncTimerTask.isDone()) { syncTimerTask = new FutureTimerTask("ltsdb-dblog-sync-timertask", syncCallable); syncTimer.schedule(syncTimerTask, storeConfig.getDbLogFlushInterval());
/** * 删除多余的快照数目 */ private void deleteOverSnapshot() throws IOException { String[] indexFiles = getIndexFiles(); if (indexFiles == null || indexFiles.length == 0) { return; } if (storeConfig.getMaxIndexSnapshotSize() > 1 && indexFiles.length > storeConfig.getMaxIndexSnapshotSize()) { for (int i = 0; i < indexFiles.length - storeConfig.getMaxIndexSnapshotSize(); i++) { FileUtils.delete(new File(storeConfig.getIndexPath(), indexFiles[i])); LOGGER.info("delete index snapshot [" + indexFiles[i] + "] succeed"); } } } }
@Override public void run() { DataBlock.LOGGER.info(this.getServiceName() + " service started"); while (!this.isStopped()) { try { if (storeConfig.isEnableFlushDataInterval()) { Thread.sleep(storeConfig.getFlushDataInterval()); } else { waitForRunning(storeConfig.getFlushDataInterval()); } // 刷到磁盘 flushDisk(); } catch (Exception e) { DataBlock.LOGGER.error(this.getServiceName() + " error:" + e.getMessage(), e); } } } }
private String[] getIndexFiles() throws IOException { String[] indexFiles = storeConfig.getIndexPath().list(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.endsWith(".snapshot"); } }); if (indexFiles == null) { throw new IOException("can't list file in " + storeConfig.getIndexPath()); } if (indexFiles.length == 0) { return null; } Arrays.sort(indexFiles, new Comparator<String>() { @Override public int compare(String left, String right) { return left.compareTo(right); } }); return indexFiles; }
@Override public void run() { try { // 检查一下当改变的量达到一定量时要snapshot if (currentChangeNum.get() - lastSnapshotChangeNum.get() > storeConfig.getIndexSnapshotThreshold()) { indexSnapshot.snapshot(); } } catch (Throwable t) { LOGGER.error("SNAPSHOT Error", t); } } }, 3, 2, TimeUnit.SECONDS);
public void init() throws DBException { try { FileUtils.createDirIfNotExist(storeConfig.getDbPath()); storeTxLogEngine.init(); dataBlockEngine.init(); indexSnapshot.init(); } catch (Exception e) { throw new DBException("DB init error:" + e.getMessage(), e); } }
public DataBlockEngine(StoreSerializer serializer, StoreConfig storeConfig) { this.serializer = serializer; this.storeConfig = storeConfig; this.dataPath = storeConfig.getDataPath(); }