public void calculateNextfileID(final List<JournalFile> files) { for (JournalFile file : files) { final long fileIdFromFile = file.getFileID(); final long fileIdFromName = getFileNameID(file.getFile().getFileName()); // The compactor could create a fileName but use a previously assigned ID. // Because of that we need to take both parts into account setNextFileID(Math.max(fileIdFromName, fileIdFromFile)); } }
@Override public String debug() throws Exception { reclaimer.scan(getDataFiles()); StringBuilder builder = new StringBuilder(); for (JournalFile file : filesRepository.getDataFiles()) { builder.append("DataFile:" + file + " posCounter = " + file.getPosCount() + " reclaimStatus = " + file.isCanReclaim() + " live size = " + file.getLiveSize() + "\n"); if (file instanceof JournalFileImpl) { builder.append(((JournalFileImpl) file).debug()); } } for (JournalFile file : filesRepository.getFreeFiles()) { builder.append("FreeFile:" + file + "\n"); } if (currentFile != null) { builder.append("CurrentFile:" + currentFile + " posCounter = " + currentFile.getPosCount() + "\n"); if (currentFile instanceof JournalFileImpl) { builder.append(((JournalFileImpl) currentFile).debug()); } } else { builder.append("CurrentFile: No current file at this point!"); } return builder.toString(); }
public JournalRecord(final JournalFile addFile, final int size) { this.addFile = addFile; this.size = size; addFile.incPosCount(); addFile.addSize(size); }
void delete(final JournalFile file) { file.incNegCount(addFile); addFile.decSize(size); if (updateFiles != null) { for (Pair<JournalFile, Integer> updFile : updateFiles) { file.incNegCount(updFile.getA()); updFile.getA().decSize(updFile.getB()); } } }
if (!currentFile.isNegReclaimCriteria()) { boolean outstandingNeg = false; if (!file.isCanReclaim() && currentFile.getNegCount(file) != 0) { logger.tracef("%s can't be reclaimed because %s has negative values", currentFile, file); outstandingNeg = true; continue; // Move to next file as we already know that this file can't be reclaimed because criterion 2) } else { currentFile.setNegReclaimCriteria(); if (!currentFile.isPosReclaimCriteria()) { int negCount = 0, posCount = currentFile.getPosCount(); logger.tracef("posCount on %s = %d", currentFile, posCount); int toNeg = files[j].getNegCount(currentFile); negCount += toNeg; } else { logger.tracef("%s can be reclaimed", currentFile); currentFile.setPosReclaimCriteria();
encoder.setFileID(currentFile.getRecordID()); currentFile.getFile().write(encoder, false, callback); } else { currentFile.getFile().write(encoder, false);
final JournalReaderCallback reader, final AtomicReference<ByteBuffer> wholeFileBufferReference) throws Exception { file.getFile().open(1, false); ByteBuffer wholeFileBuffer = null; try { final int filesize = (int) file.getFile().size(); final int journalFileSize = file.getFile().read(wholeFileBuffer); if (readFileId != file.getRecordID()) { wholeFileBuffer.position(pos + 1); continue; if (file.getJournalVersion() >= 2) { if (JournalImpl.isInvalidSize(journalFileSize, wholeFileBuffer.position(), DataConstants.SIZE_BYTE)) { reader.markAsDataFile(file); int recordSize = JournalImpl.getRecordSize(recordType, file.getJournalVersion()); recordType + " file:" + file.getFile().getFileName() + " recordSize: " + recordSize + recordID + " file:" + file.getFile().getFileName() + " is corrupted and it is being ignored (III)");
private void setupPosNeg(final int fileNumber, final int pos, final int... neg) { JournalFile file = files[fileNumber]; int totalDep = file.getTotalNegativeToOthers(); for (int i = 0; i < pos; i++) { file.incPosCount(); } for (int i = 0; i < neg.length; i++) { JournalFile reclaimable2 = files[i]; for (int j = 0; j < neg[i]; j++) { file.incNegCount(reclaimable2); totalDep++; } } assertEquals(totalDep, file.getTotalNegativeToOthers()); }
/** * @param jf * @return */ private Pair<Long, Integer> getPair(JournalFile jf) { return new Pair<>(jf.getFileID(), jf.getPosCount()); }
public ReplicationStartSyncMessage(JournalFile[] datafiles, AbstractJournalStorageManager.JournalContent contentType, String nodeID, boolean allowsAutoFailBack) { this(); this.nodeID = nodeID; this.allowsAutoFailBack = allowsAutoFailBack; synchronizationIsFinished = false; ids = new long[datafiles.length]; for (int i = 0; i < datafiles.length; i++) { ids[i] = datafiles[i].getFileID(); } switch (contentType) { case MESSAGES: dataType = SyncDataType.JournalMessages; break; case BINDINGS: dataType = SyncDataType.JournalBindings; break; default: throw new IllegalArgumentException(); } }
private boolean needsCompact() throws Exception { JournalFile[] dataFiles = getDataFiles(); long totalLiveSize = 0; for (JournalFile file : dataFiles) { totalLiveSize += file.getLiveSize(); } long totalBytes = dataFiles.length * (long) fileSize; long compactMargin = (long) (totalBytes * compactPercentage); boolean needCompact = totalLiveSize < compactMargin && dataFiles.length > compactMinFiles; return needCompact; }
private void assertCanDelete(final int... fileNumber) { for (int num : fileNumber) { Assert.assertTrue(files[num].isCanReclaim()); } }
/** * The caller of this method needs to guarantee appendLock.lock before calling this method if being used outside of the lock context. * or else potFilesMap could be affected */ public void rollback(final JournalFile file) { JournalCompactor compactor = journal.getCompactor(); if (compacting && compactor != null) { compactor.addCommandRollback(this, file); } else { // Now add negs for the pos we added in each file in which there were // transactional operations // Note that we do this on rollback as we do on commit, since we need // to ensure the file containing // the rollback record doesn't get deleted before the files with the // transactional operations are deleted // Otherwise we may run into problems especially with XA where we are // just left with a prepare when the tx // has actually been rolled back for (JournalFile jf : pendingFiles) { file.incNegCount(jf); } } }
protected void writeEncoder(final JournalInternalRecord record) throws Exception { record.setFileID(currentFile.getRecordID()); record.encode(getWritingChannel()); }
private void addFile(final JournalFile file) { if (pendingFiles == null) { pendingFiles = new HashSet<>(); } if (!pendingFiles.contains(file)) { pendingFiles.add(file); // We add a pos for the transaction itself in the file - this // prevents any transactional operations // being deleted before a commit or rollback is written file.incPosCount(); } }
@Test public void testDeleteWhileCleanup() throws Exception { setup(2, 60 * 1024, false); createJournal(); startJournal(); load(); for (int i = 0; i < 100; i++) { add(i); } journal.forceMoveNextFile(); for (int i = 10; i < 90; i++) { delete(i); } startCompact(); // Delete part of the live records while cleanup still working for (int i = 1; i < 5; i++) { delete(i); } finishCompact(); // Delete part of the live records after cleanup is done for (int i = 5; i < 10; i++) { delete(i); } assertEquals(9, journal.getCurrentFile().getNegCount(journal.getDataFiles()[0])); journal.forceMoveNextFile(); stopJournal(); createJournal(); startJournal(); loadAndCheck(); }
if (!currentFile.isNegReclaimCriteria()) { boolean outstandingNeg = false; if (!file.isCanReclaim() && currentFile.getNegCount(file) != 0) { logger.tracef("%s can't be reclaimed because %s has negative values", currentFile, file); outstandingNeg = true; continue; // Move to next file as we already know that this file can't be reclaimed because criterion 2) } else { currentFile.setNegReclaimCriteria(); if (!currentFile.isPosReclaimCriteria()) { int negCount = 0, posCount = currentFile.getPosCount(); logger.tracef("posCount on %s = %d", currentFile, posCount); int toNeg = files[j].getNegCount(currentFile); negCount += toNeg; } else { logger.tracef("%s can be reclaimed", currentFile); currentFile.setPosReclaimCriteria();
encoder.setFileID(currentFile.getRecordID()); currentFile.getFile().write(encoder, false, callback); } else { currentFile.getFile().write(encoder, false);
final JournalReaderCallback reader, final AtomicReference<ByteBuffer> wholeFileBufferReference) throws Exception { file.getFile().open(1, false); ByteBuffer wholeFileBuffer = null; try { final int filesize = (int) file.getFile().size(); final int journalFileSize = file.getFile().read(wholeFileBuffer); if (readFileId != file.getRecordID()) { wholeFileBuffer.position(pos + 1); continue; if (file.getJournalVersion() >= 2) { if (JournalImpl.isInvalidSize(journalFileSize, wholeFileBuffer.position(), DataConstants.SIZE_BYTE)) { reader.markAsDataFile(file); int recordSize = JournalImpl.getRecordSize(recordType, file.getJournalVersion()); recordType + " file:" + file.getFile().getFileName() + " recordSize: " + recordSize + recordID + " file:" + file.getFile().getFileName() + " is corrupted and it is being ignored (III)");