public String getShortDescription() { StringBuilder sb = new StringBuilder(); sb.append("{"); sb.append("ID=" + backupId).append(","); sb.append("Type=" + getType()).append(","); sb.append("Tables=" + getTableListAsString()).append(","); sb.append("State=" + getState()).append(","); Date date = null; Calendar cal = Calendar.getInstance(); cal.setTimeInMillis(getStartTs()); date = cal.getTime(); sb.append("Start time=" + date).append(","); if (state == BackupState.FAILED) { sb.append("Failed message=" + getFailedMsg()).append(","); } else if (state == BackupState.RUNNING) { sb.append("Phase=" + getPhase()).append(","); } else if (state == BackupState.COMPLETE) { cal = Calendar.getInstance(); cal.setTimeInMillis(getCompleteTs()); date = cal.getTime(); sb.append("End time=" + date).append(","); } sb.append("Progress=" + getProgress() + "%"); sb.append("}"); return sb.toString(); }
public BackupInfo toBackupInfo() { BackupInfo info = new BackupInfo(); info.setType(backupImage.getType()); List<TableName> list = backupImage.getTableNames(); TableName[] tables = new TableName[list.size()]; info.addTables(list.toArray(tables)); info.setBackupId(backupImage.getBackupId()); info.setStartTs(backupImage.getStartTs()); info.setBackupRootDir(backupImage.getRootDir()); if (backupImage.getType() == BackupType.INCREMENTAL) { info.setHLogTargetDir(BackupUtils.getLogBackupDir(backupImage.getRootDir(), backupImage.getBackupId())); } return info; } }
/** * Construct manifest for a ongoing backup. * @param backup The ongoing backup info */ public BackupManifest(BackupInfo backup) { BackupImage.Builder builder = BackupImage.newBuilder(); this.backupImage = builder.withBackupId(backup.getBackupId()).withType(backup.getType()) .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames()) .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build(); }
protected Path getBulkOutputDir() { String backupId = backupInfo.getBackupId(); Path path = new Path(backupInfo.getBackupRootDir()); path = new Path(path, ".tmp"); path = new Path(path, backupId); return path; } }
public String getStatusAndProgressAsString() { StringBuilder sb = new StringBuilder(); sb.append("id: ").append(getBackupId()).append(" state: ").append(getState()) .append(" progress: ").append(getProgress()); return sb.toString(); }
private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long startTime) throws IOException { List<BackupInfo> history = table.getBackupHistory(); for (BackupInfo info : history) { List<TableName> tables = info.getTableNames(); if (!tables.contains(tn)) { continue; } return info.getStartTs() <= startTime; } return false; }
public static BackupInfo fromProto(BackupProtos.BackupInfo proto) { BackupInfo context = new BackupInfo(); context.setBackupId(proto.getBackupId()); context.setBackupTableInfoMap(toMap(proto.getBackupTableInfoList())); context.setCompleteTs(proto.getCompleteTs()); if (proto.hasFailedMessage()) { context.setFailedMsg(proto.getFailedMessage()); } if (proto.hasBackupState()) { context.setState(BackupInfo.BackupState.valueOf(proto.getBackupState().name())); } context.setHLogTargetDir(BackupUtils.getLogBackupDir(proto.getBackupRootDir(), proto.getBackupId())); if (proto.hasBackupPhase()) { context.setPhase(BackupPhase.valueOf(proto.getBackupPhase().name())); } if (proto.hasProgress()) { context.setProgress(proto.getProgress()); } context.setStartTs(proto.getStartTs()); context.setBackupRootDir(proto.getBackupRootDir()); context.setType(BackupType.valueOf(proto.getBackupType().name())); context.setWorkers(proto.getWorkersNumber()); context.setBandwidth(proto.getBandwidth()); return context; }
public BackupProtos.BackupInfo toProtosBackupInfo() { BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder(); builder.setBackupId(getBackupId()); setBackupTableInfoMap(builder); builder.setCompleteTs(getCompleteTs()); if (getFailedMsg() != null) { builder.setFailedMessage(getFailedMsg()); } if (getState() != null) { builder.setBackupState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name())); } if (getPhase() != null) { builder.setBackupPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name())); } builder.setProgress(getProgress()); builder.setStartTs(getStartTs()); builder.setBackupRootDir(getBackupRootDir()); builder.setBackupType(BackupProtos.BackupType.valueOf(getType().name())); builder.setWorkersNumber(workers); builder.setBandwidth(bandwidth); return builder.build(); }
backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime()); backupInfo.setFailedMsg(e.getMessage()); backupInfo.setState(BackupState.FAILED); "BackupId=" + backupInfo.getBackupId() + ",startts=" + backupInfo.getStartTs() + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase=" + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg(); LOG.error(backupFailedData); cleanupAndRestoreBackupSystem(conn, backupInfo, conf); LOG.error("Backup " + backupInfo.getBackupId() + " failed."); } catch (IOException ee) { LOG.error("Please run backup repair tool manually to restore backup system integrity");
backupInfo.setPhase(BackupPhase.PREPARE_INCREMENTAL); LOG.debug("For incremental backup, current table set is " + backupManager.getIncrementalBackupTableSet()); backupInfo.getBackupRootDir()); backupManager.recordWALFiles(backupInfo.getIncrBackupFileList()); } catch (Exception e) { String msg = "Unexpected exception in incremental-backup: incremental copy " + backupId; backupInfo.setIncrTimestampMap(previousTimestampMap); backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps); backupManager.writeBackupStartCode(newStartCode); handleBulkLoad(backupInfo.getTableNames());
backupRoot = bInfo.getBackupRootDir(); } else if (!bInfo.getBackupRootDir().equals(backupRoot)) { throw new IOException("Found different backup destinations in a list of a backup sessions " + "\n1. " + backupRoot + "\n" + "2. " + bInfo.getBackupRootDir()); if (bInfo.getType() == BackupType.FULL) { throw new IOException("FULL backup image can not be merged for: \n" + bInfo); if (bInfo.getState() != BackupState.COMPLETE) { throw new IOException("Backup image " + backupId + " can not be merged becuase of its state: " + bInfo.getState()); allTables.addAll(bInfo.getTableNames()); long time = bInfo.getStartTs(); if (time < minTime) { minTime = time; BackupInfo.Filter destinationFilter = info -> info.getBackupRootDir().equals(backupDest); long time = info.getStartTs(); return time >= startRangeTime && time <= endRangeTime ; }; List<TableName> tables = info.getTableNames(); return !Collections.disjoint(allTables, tables); }; BackupInfo.Filter typeFilter = info -> info.getType() == BackupType.INCREMENTAL; BackupInfo.Filter stateFilter = info -> info.getState() == BackupState.COMPLETE;
private List<BackupInfo> getAffectedBackupSessions(BackupInfo backupInfo, TableName tn, BackupSystemTable table) throws IOException { LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn); long ts = backupInfo.getStartTs(); List<BackupInfo> list = new ArrayList<>(); List<BackupInfo> history = table.getBackupHistory(backupInfo.getBackupRootDir()); // Scan from most recent to backupInfo // break when backupInfo reached for (BackupInfo info : history) { if (info.getStartTs() == ts) { break; } List<TableName> tables = info.getTableNames(); if (tables.contains(tn)) { BackupType bt = info.getType(); if (bt == BackupType.FULL) { // Clear list if we encounter FULL backup list.clear(); } else { LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn + " added " + info.getBackupId() + " tables=" + info.getTableListAsString()); list.add(info); } } } return list; }
BackupManager backupManager, BackupType type, Configuration conf) throws IOException { backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime()); backupInfo.setState(BackupState.COMPLETE); backupInfo.setProgress(100); obtainBackupMetaDataStr(backupInfo) + ",startts=" + backupInfo.getStartTs() + ",completets=" + backupInfo.getCompleteTs() + ",bytescopied=" + backupInfo.getTotalBytesCopied(); if (LOG.isDebugEnabled()) { LOG.debug("Backup " + backupInfo.getBackupId() + " finished: " + backupCompleteData); LOG.info("Backup " + backupInfo.getBackupId() + " completed.");
props.put("backupRoot", backupInfo.getBackupRootDir()); admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE, LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props); backupInfo.setPhase(BackupPhase.SNAPSHOT); for (TableName tableName : tableList) { String snapshotName = backupInfo.setSnapshotName(tableName, snapshotName); snapshotCopy(backupInfo); backupManager.addIncrementalBackupTableSet(backupInfo.getTables()); backupInfo.setState(BackupState.COMPLETE); backupManager.writeRegionServerLogTimestamp(backupInfo.getTables(), newTimestamps);
backupInfo.setPhase(BackupPhase.SNAPSHOTCOPY); float numOfSnapshots = backupInfo.getSnapshotNames().size(); for (TableName table : backupInfo.getTables()) { String[] args = new String[4]; args[0] = "-snapshot"; args[1] = backupInfo.getSnapshotName(table); args[2] = "-copy-to"; args[3] = backupInfo.getTableBackupDir(table); String jobname = "Full-Backup_" + backupInfo.getBackupId() + "_" + table.getNameAsString(); if (LOG.isDebugEnabled()) { LOG.debug("Setting snapshot copy job name to : " + jobname);
/** * Begin the overall backup. * @param backupInfo backup info * @throws IOException exception */ protected void beginBackup(BackupManager backupManager, BackupInfo backupInfo) throws IOException { BackupSystemTable.snapshot(conn); backupManager.setBackupInfo(backupInfo); // set the start timestamp of the overall backup long startTs = EnvironmentEdgeManager.currentTime(); backupInfo.setStartTs(startTs); // set overall backup status: ongoing backupInfo.setState(BackupState.RUNNING); backupInfo.setPhase(BackupPhase.REQUEST); LOG.info("Backup " + backupInfo.getBackupId() + " started at " + startTs + "."); backupManager.updateBackupInfo(backupInfo); if (LOG.isDebugEnabled()) { LOG.debug("Backup session " + backupInfo.getBackupId() + " has been started."); } }
+ backupInfo.getPhase()); if (backupInfo.getPhase().equals(BackupPhase.SNAPSHOTCOPY) || backupInfo.getPhase().equals(BackupPhase.INCREMENTAL_COPY) || backupInfo.getPhase().equals(BackupPhase.STORE_MANIFEST)) { FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf); for (TableName table : backupInfo.getTables()) { Path targetDirPath = new Path(HBackupFileSystem.getTableBackupDir(backupInfo.getBackupRootDir(), backupInfo.getBackupId(), table)); if (outputFs.delete(targetDirPath, true)) { LOG.debug("Cleaning up uncompleted backup data at " + targetDirPath.toString() LOG.error("Cleaning up uncompleted backup data of " + backupInfo.getBackupId() + " at " + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + ".");
LOG.info("Deleting backup " + backupInfo.getBackupId() + " ..."); List<TableName> tables = backupInfo.getTableNames(); long startTime = backupInfo.getStartTs(); for (TableName tn : tables) { boolean isLastBackupSession = isLastBackupSession(sysTable, tn, startTime); if (info.equals(backupInfo)) { continue; Path p = new Path(f); try { LOG.debug("Delete backup info " + p + " for " + backupInfo.getBackupId()); if (!fs.delete(p)) { if (fs.exists(p)) { sysTable.deleteBackupInfo(backupInfo.getBackupId()); LOG.info("Delete backup " + backupInfo.getBackupId() + " completed."); totalDeleted++; } else {
/** * Get backup request meta data dir as string. * @param backupInfo backup info * @return meta data dir */ protected String obtainBackupMetaDataStr(BackupInfo backupInfo) { StringBuffer sb = new StringBuffer(); sb.append("type=" + backupInfo.getType() + ",tablelist="); for (TableName table : backupInfo.getTables()) { sb.append(table + ";"); } if (sb.lastIndexOf(";") > 0) { sb.delete(sb.lastIndexOf(";"), sb.lastIndexOf(";") + 1); } sb.append(",targetRootDir=" + backupInfo.getBackupRootDir()); return sb.toString(); }
/** * Read the last backup start code (timestamp) of last successful backup. Will return null if * there is no startcode stored in backup system table or the value is of length 0. These two * cases indicate there is no successful backup completed so far. * @return the timestamp of a last successful backup * @throws IOException exception */ public String readBackupStartCode() throws IOException { return systemTable.readBackupStartCode(backupInfo.getBackupRootDir()); }