/** * Remove all logs with logId <= {@code lastLogId}. */ private void removeAllLogs(long lastLogId, String why) { if (logs.size() <= 1) { return; } LOG.info("Remove all state logs with ID less than {}, since {}", lastLogId, why); boolean removed = false; while (logs.size() > 1) { ProcedureWALFile log = logs.getFirst(); if (lastLogId < log.getLogId()) { break; } removeLogFile(log, walArchiveDir); removed = true; } if (removed) { buildHoldingCleanupTracker(); } }
final ProcedureWALFile log = new ProcedureWALFile(fs, logFile); if (logFile.getLen() == 0) { LOG.warn("Remove uninitialized log: {}", logFile); log.removeFile(walArchiveDir); return null; log.open(); } catch (ProcedureWALFormat.InvalidWALDataException e) { LOG.warn("Remove uninitialized log: {}", logFile, e); log.removeFile(walArchiveDir); return null; } catch (IOException e) { log.readTracker(); } catch (IOException e) { log.getTracker().reset(); log.getTracker().setPartialFlag(true); LOG.warn("Unable to read tracker for {}", log, e); log.close(); return log;
@Override public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof ProcedureWALFile)) return false; return compareTo((ProcedureWALFile)o) == 0; }
public void processProcedureWALFile(ProcedureWALFile log) throws IOException { log.open(); ProcedureWALHeader header = log.getHeader(); printHeader(header); FSDataInputStream stream = log.getStream(); try { boolean hasMore = true; while (hasMore) { ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); if (entry == null) { out.println("No more entry, exiting with missing EOF"); hasMore = false; break; } switch (entry.getType()) { case PROCEDURE_WAL_EOF: hasMore = false; break; default: printEntry(entry); } } } catch (IOException e) { out.println("got an exception while reading the procedure WAL " + e.getMessage()); } finally { log.close(); } }
public void read(ProcedureWALFile log) throws IOException { localTracker = log.getTracker(); if (localTracker.isPartial()) { LOG.info("Rebuilding tracker for {}", log); FSDataInputStream stream = log.getStream(); try { boolean hasMore = true; log.setProcIds(localProcedureMap.getMinModifiedProcId(), localProcedureMap.getMaxModifiedProcId()); if (localTracker.isPartial()) {
private void buildHoldingCleanupTracker() { if (logs.size() <= 1) { // we only have one wal, so nothing to do holdingCleanupTracker.reset(); return; } // compute the holding tracker. // - the first WAL is used for the 'updates' // - the global tracker will be used to determine whether a procedure has been deleted // - other trackers will be used to determine whether a procedure has been updated, as a deleted // procedure can always be detected by checking the global tracker, we can save the deleted // checks when applying other trackers holdingCleanupTracker.resetTo(logs.getFirst().getTracker(), true); holdingCleanupTracker.setDeletedIfDeletedByThem(storeTracker); // the logs is a linked list, so avoid calling get(index) on it. Iterator<ProcedureWALFile> iter = logs.iterator(); // skip the tracker for the first file when creating the iterator. iter.next(); ProcedureStoreTracker tracker = iter.next().getTracker(); // testing iter.hasNext after calling iter.next to skip applying the tracker for last file, // which is just the storeTracker above. while (iter.hasNext()) { holdingCleanupTracker.setDeletedIfModifiedInBoth(tracker); if (holdingCleanupTracker.isEmpty()) { break; } tracker = iter.next().getTracker(); } }
private ProcedureWALFile initOldLog(final FileStatus logFile) throws IOException { ProcedureWALFile log = new ProcedureWALFile(fs, logFile); if (logFile.getLen() == 0) { LOG.warn("Remove uninitialized log: " + logFile); log.removeFile(); return null; log.open(); } catch (ProcedureWALFormat.InvalidWALDataException e) { LOG.warn("Remove uninitialized log: " + logFile, e); log.removeFile(); return null; } catch (IOException e) { if (log.isCompacted()) { try { log.readTrailer(); } catch (IOException e) { LOG.warn("Unfinished compacted log: " + logFile, e); log.removeFile(); return null;
ProcedureWALFile pwf = procedureWALFiles.get(i); out.write("\n <tr>\n <td> "); out.print( pwf.getLogId() ); out.write("</td>\n <td> "); out.print( TraditionalBinaryPrefix.long2String(pwf.getSize(), "B", 1) ); out.write(" </td>\n <td> "); out.print( new Date(pwf.getTimestamp()) ); out.write(" </td>\n <td> "); out.print( escapeXml(pwf.toString()) ); out.write(" </td>\n </tr>\n "); for (ProcedureWALFile cwf:corruptedWALFiles) { out.write("\n <tr>\n <td> "); out.print( cwf.getLogId() ); out.write("</td>\n <td> "); out.print( TraditionalBinaryPrefix.long2String(cwf.getSize(), "B", 1) ); out.write(" </td>\n <td> "); out.print( new Date(cwf.getTimestamp()) ); out.write(" </td>\n <td> "); out.print( escapeXml(cwf.toString()) ); out.write(" </td>\n </tr>\n ");
private boolean removeLogFile(final ProcedureWALFile log, final Path walArchiveDir) { try { LOG.trace("Removing log={}", log); log.removeFile(walArchiveDir); logs.remove(log); LOG.debug("Removed log={}, activeLogs={}", log, logs); assert logs.size() > 0 : "expected at least one log"; } catch (IOException e) { LOG.error("Unable to remove log: " + log, e); return false; } return true; }
public void removeFile(final Path walArchiveDir) throws IOException { close(); boolean archived = false; if (walArchiveDir != null) { Path archivedFile = new Path(walArchiveDir, logFile.getName()); LOG.info("Archiving " + logFile + " to " + archivedFile); if (!fs.rename(logFile, archivedFile)) { LOG.warn("Failed archive of " + logFile + ", deleting"); } else { archived = true; } } if (!archived) { if (!fs.delete(logFile, false)) { LOG.warn("Failed delete of " + logFile); } } }
/** * Reads a log file and outputs its contents. * * @param conf HBase configuration relevant to this log file * @param p path of the log file to be read * @throws IOException IOException */ public void processFile(final Configuration conf, final Path p) throws IOException { FileSystem fs = p.getFileSystem(conf); if (!fs.exists(p)) { System.err.println("ERROR, file doesnt exist: " + p); return; } if (!fs.isFile(p)) { System.err.println(p + " is not a file"); return; } FileStatus logFile = fs.getFileStatus(p); if (logFile.getLen() == 0) { out.println("Zero length file: " + p); return; } out.println("Opening procedure state-log: " + p); ProcedureWALFile log = new ProcedureWALFile(fs, logFile); processProcedureWALFile(log); }
public void read(ProcedureWALFile log, ProcedureWALFormat.Loader loader) throws IOException { FSDataInputStream stream = log.getStream(); try { boolean hasMore = true; log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); procedureMap.mergeTail(localProcedureMap);
public void readTracker() throws IOException { ProcedureWALTrailer trailer = readTrailer(); try { stream.seek(trailer.getTrackerPos()); final ProcedureProtos.ProcedureStoreTracker trackerProtoBuf = ProcedureProtos.ProcedureStoreTracker.parseDelimitedFrom(stream); tracker.resetToProto(trackerProtoBuf); } finally { stream.seek(startPos); } }
private void initTrackerFromOldLogs() { // TODO: Load the most recent tracker available if (!logs.isEmpty()) { ProcedureWALFile log = logs.getLast(); try { log.readTracker(storeTracker); } catch (IOException e) { LOG.warn("Unable to read tracker for " + log + " - " + e.getMessage()); // try the next one... storeTracker.reset(); storeTracker.setPartialFlag(true); } } }
public void processProcedureWALFile(ProcedureWALFile log) throws IOException { log.open(); ProcedureWALHeader header = log.getHeader(); printHeader(header); FSDataInputStream stream = log.getStream(); try { boolean hasMore = true; while (hasMore) { ProcedureWALEntry entry = ProcedureWALFormat.readEntry(stream); if (entry == null) { out.println("No more entry, exiting with missing EOF"); hasMore = false; break; } switch (entry.getType()) { case PROCEDURE_WAL_EOF: hasMore = false; break; default: printEntry(entry); } } } catch (IOException e) { out.println("got an exception while reading the procedure WAL " + e.getMessage()); } finally { log.close(); } }
public void read(final ProcedureWALFile log) throws IOException { localTracker = log.getTracker().isPartial() ? log.getTracker() : null; if (localTracker != null) { LOG.info("Rebuilding tracker for " + log); FSDataInputStream stream = log.getStream(); try { boolean hasMore = true; log.setProcIds(localProcedureMap.getMinProcId(), localProcedureMap.getMaxProcId()); procedureMap.mergeTail(localProcedureMap);
/** * If last log's tracker is not null, use it as {@link #storeTracker}. Otherwise, set storeTracker * as partial, and let {@link ProcedureWALFormatReader} rebuild it using entries in the log. */ private void initTrackerFromOldLogs() { if (logs.isEmpty() || !isRunning()) { return; } ProcedureWALFile log = logs.getLast(); if (!log.getTracker().isPartial()) { storeTracker.resetTo(log.getTracker()); } else { storeTracker.reset(); storeTracker.setPartialFlag(true); } }
public static void load(final Iterator<ProcedureWALFile> logs, final ProcedureStoreTracker tracker, final Loader loader) throws IOException { final ProcedureWALFormatReader reader = new ProcedureWALFormatReader(tracker, loader); tracker.setKeepDeletes(true); try { // Ignore the last log which is current active log. while (logs.hasNext()) { ProcedureWALFile log = logs.next(); log.open(); try { reader.read(log); } finally { log.close(); } } reader.finish(); // The tracker is now updated with all the procedures read from the logs tracker.setPartialFlag(false); tracker.resetUpdates(); } finally { tracker.setKeepDeletes(false); } }
private boolean rollWriter() throws IOException { if (!isRunning()) { return false; } // Create new state-log if (!rollWriter(flushLogId + 1)) { LOG.warn("someone else has already created log {}", flushLogId); return false; } // We have the lease on the log, // but we should check if someone else has created new files if (getMaxLogId(getLogFiles()) > flushLogId) { LOG.warn("Someone else created new logs. Expected maxLogId < {}", flushLogId); logs.getLast().removeFile(this.walArchiveDir); return false; } // We have the lease on the log return true; }