public static void main(String[] args) throws Exception { ServerUtilOpts opts = new ServerUtilOpts(); opts.parseArgs(MergeStats.class.getName(), args); try (AccumuloClient client = opts.createClient()) { Map<String,String> tableIdMap = client.tableOperations().tableIdMap(); ZooReaderWriter zooReaderWriter = opts.getServerContext().getZooReaderWriter(); for (Entry<String,String> entry : tableIdMap.entrySet()) { final String table = entry.getKey(), tableId = entry.getValue(); String path = ZooUtil.getRoot(client.getInstanceID()) + Constants.ZTABLES + "/" + tableId + "/merge"; MergeInfo info = new MergeInfo(); if (zooReaderWriter.exists(path)) { byte[] data = zooReaderWriter.getData(path, new Stat()); DataInputBuffer in = new DataInputBuffer(); in.reset(data, data.length); info.readFields(in); } System.out.println(String.format("%25s %10s %10s %s", table, info.getState(), info.getOperation(), info.getExtent())); } } } }
public MergeState nextMergeState(AccumuloClient accumuloClient, CurrentState master) throws Exception { MergeState state = info.getState(); if (state == MergeState.NONE) return state; if (total == 0) { log.trace("failed to see any tablets for this range, ignoring {}", info.getExtent()); return state; info.getExtent(), state, info.isDelete()); if (state == MergeState.STARTED) { state = MergeState.SPLITTING; if (!info.isDelete() && total == 1) { log.info("Merge range is already contained in a single tablet {}", info.getExtent()); state = MergeState.COMPLETE; } else if (hosted == total) { if (info.isDelete()) { if (!lowerSplit) log.info("Waiting for {} lower split to occur {}", info, info.getExtent()); else if (!upperSplit) log.info("Waiting for {} upper split to occur {}", info, info.getExtent()); else state = MergeState.WAITING_FOR_CHOPPED; log.info("Waiting for {} hosted tablets to be {} {}", hosted, total, info.getExtent()); log.info("{} tablets are chopped {}", chopped, info.getExtent()); if (chopped == needsToBeChopped) { state = MergeState.WAITING_FOR_OFFLINE;
public void update(KeyExtent ke, TabletState state, boolean chopped, boolean hasWALs) { if (info.getState().equals(MergeState.NONE)) return; if (!upperSplit && info.getExtent().getEndRow().equals(ke.getPrevEndRow())) { log.info("Upper split found"); upperSplit = true; } if (!lowerSplit && info.getExtent().getPrevEndRow().equals(ke.getEndRow())) { log.info("Lower split found"); lowerSplit = true; } if (!info.overlaps(ke)) return; if (info.needsToBeChopped(ke)) { this.needsToBeChopped++; if (chopped) { if (state.equals(TabletState.HOSTED)) { this.chopped++; } else if (!hasWALs) { this.chopped++; } } } this.total++; if (state.equals(TabletState.HOSTED)) this.hosted++; if (state.equals(TabletState.UNASSIGNED) || state.equals(TabletState.SUSPENDED)) this.unassigned++; }
public static void setMerges(IteratorSetting cfg, Collection<MergeInfo> merges) { DataOutputBuffer buffer = new DataOutputBuffer(); try { for (MergeInfo info : merges) { KeyExtent extent = info.getExtent(); if (extent != null && !info.getState().equals(MergeState.NONE)) { info.write(buffer); } } } catch (Exception ex) { throw new RuntimeException(ex); } String encoded = Base64.getEncoder() .encodeToString(Arrays.copyOf(buffer.getData(), buffer.getLength())); cfg.addOption(MERGES_OPTION, encoded); }
private Map<Table.ID,MergeInfo> parseMerges(String merges) { if (merges == null) return null; try { Map<Table.ID,MergeInfo> result = new HashMap<>(); DataInputBuffer buffer = new DataInputBuffer(); byte[] data = Base64.getDecoder().decode(merges); buffer.reset(data, data.length); while (buffer.available() > 0) { MergeInfo mergeInfo = new MergeInfo(); mergeInfo.readFields(buffer); result.put(mergeInfo.extent.getTableId(), mergeInfo); } return result; } catch (Exception ex) { throw new RuntimeException(ex); } }
@Override public Repo<Master> call(long tid, Master env) throws Exception { if (RootTable.ID.equals(tableId) && Operation.MERGE.equals(op)) { log.warn("Attempt to merge tablets for {} does nothing. It is not splittable.", RootTable.NAME); } Text start = startRow.length == 0 ? null : new Text(startRow); Text end = endRow.length == 0 ? null : new Text(endRow); if (start != null && end != null) if (start.compareTo(end) >= 0) throw new AcceptableThriftTableOperationException(tableId.canonicalID(), null, TableOperation.MERGE, TableOperationExceptionType.BAD_RANGE, "start row must be less than end row"); env.mustBeOnline(tableId); MergeInfo info = env.getMergeInfo(tableId); if (info.getState() == MergeState.NONE) { KeyExtent range = new KeyExtent(tableId, end, start); env.setMergeState(new MergeInfo(range, op), MergeState.STARTED); } return new TableRangeOpWait(namespaceId, tableId); }
public MergeState nextMergeState(Connector connector, CurrentState master) throws Exception { MergeState state = info.getState(); if (state == MergeState.NONE) return state; if (total == 0) { log.trace("failed to see any tablets for this range, ignoring " + info.getRange()); return state; log.info("Computing next merge state for " + info.getRange() + " which is presently " + state + " isDelete : " + info.isDelete()); if (state == MergeState.STARTED) { state = MergeState.SPLITTING; if (!info.isDelete() && total == 1) { log.info("Merge range is already contained in a single tablet " + info.getRange()); state = MergeState.COMPLETE; } else if (hosted == total) { if (info.isDelete()) { if (!lowerSplit) log.info("Waiting for " + info + " lower split to occur " + info.getRange()); else if (!upperSplit) log.info("Waiting for " + info + " upper split to occur " + info.getRange()); else state = MergeState.WAITING_FOR_CHOPPED; log.info("Waiting for " + hosted + " hosted tablets to be " + total + " " + info.getRange()); log.info(chopped + " tablets are chopped " + info.getRange()); if (chopped == needsToBeChopped) { state = MergeState.WAITING_FOR_OFFLINE; } else {
public void setMergeState(MergeInfo info, MergeState state) throws KeeperException, InterruptedException { synchronized (mergeLock) { String path = getZooKeeperRoot() + Constants.ZTABLES + "/" + info.getExtent().getTableId() + "/merge"; info.setState(state); if (state.equals(MergeState.NONE)) { context.getZooReaderWriter().recursiveDelete(path, NodeMissingPolicy.SKIP); } else { DataOutputBuffer out = new DataOutputBuffer(); try { info.write(out); } catch (IOException ex) { throw new RuntimeException("Unlikely", ex); } context.getZooReaderWriter().putPersistentData(path, out.getData(), state.equals(MergeState.STARTED) ? ZooUtil.NodeExistsPolicy.FAIL : ZooUtil.NodeExistsPolicy.OVERWRITE); } mergeLock.notifyAll(); } nextEvent.event("Merge state of %s set to %s", info.getExtent(), state); }
public void update(KeyExtent ke, TabletState state, boolean chopped, boolean hasWALs) { if (ke.isRootTablet()) return; if (info.getState().equals(MergeState.NONE)) return; if (!upperSplit && info.getRange().getEndRow().equals(ke.getPrevEndRow())) { log.info("Upper split found"); upperSplit = true; if (!lowerSplit && info.getRange().getPrevEndRow().equals(ke.getEndRow())) { log.info("Lower split found"); lowerSplit = true; if (!info.overlaps(ke)) return; if (info.needsToBeChopped(ke)) { this.needsToBeChopped++; if (chopped) {
private void sendChopRequest(MergeInfo info, TabletState state, TabletLocationState tls) { // Don't bother if we're in the wrong state if (!info.getState().equals(MergeState.WAITING_FOR_CHOPPED)) return; // Tablet must be online if (!state.equals(TabletState.HOSTED)) return; // Tablet isn't already chopped if (tls.chopped) return; // Tablet ranges intersect if (info.needsToBeChopped(tls.extent)) { TServerConnection conn; try { conn = this.master.tserverSet.getConnection(tls.current); if (conn != null) { Master.log.info("Asking {} to chop {}", tls.current, tls.extent); conn.chop(this.master.masterLock, tls.extent); } else { Master.log.warn("Could not connect to server {}", tls.current); } } catch (TException e) { Master.log.warn("Communications error asking tablet server to chop a tablet"); } } }
public static void setMerges(IteratorSetting cfg, Collection<MergeInfo> merges) { DataOutputBuffer buffer = new DataOutputBuffer(); try { for (MergeInfo info : merges) { KeyExtent extent = info.getRange(); if (extent != null && !info.getState().equals(MergeState.NONE)) { info.write(buffer); } } } catch (Exception ex) { throw new RuntimeException(ex); } String encoded = new String(Base64.encodeBase64(Arrays.copyOf(buffer.getData(), buffer.getLength())), UTF_8); cfg.addOption(MERGES_OPTION, encoded); }
public void setMergeState(MergeInfo info, MergeState state) throws IOException, KeeperException, InterruptedException { synchronized (mergeLock) { String path = ZooUtil.getRoot(instance.getInstanceID()) + Constants.ZTABLES + "/" + info.getRange().getTableId().toString() + "/merge"; info.setState(state); if (state.equals(MergeState.NONE)) { ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP); } else { DataOutputBuffer out = new DataOutputBuffer(); try { info.write(out); } catch (IOException ex) { throw new RuntimeException("Unlikely", ex); } ZooReaderWriter.getInstance().putPersistentData(path, out.getData(), state.equals(MergeState.STARTED) ? ZooUtil.NodeExistsPolicy.FAIL : ZooUtil.NodeExistsPolicy.OVERWRITE); } mergeLock.notifyAll(); } nextEvent.event("Merge state of %s set to %s", info.getRange(), state); }
Map<Table.ID,MergeStats> currentMerges = new HashMap<>(); for (MergeInfo merge : master.merges()) { if (merge.getExtent() != null) { currentMerges.put(merge.getExtent().getTableId(), new MergeStats(merge)); mergeStats = currentMerges.get(tableId); if (mergeStats == null) { mergeStats = new MergeStats(new MergeInfo());
@Override public long isReady(long tid, Master env) { if (!env.getMergeInfo(tableId).getState().equals(MergeState.NONE)) { return 50; } return 0; }
private boolean verifyMergeConsistency(AccumuloClient accumuloClient, CurrentState master) throws TableNotFoundException, IOException { MergeStats verify = new MergeStats(info); KeyExtent extent = info.getExtent(); Scanner scanner = accumuloClient .createScanner(extent.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY); if (!tls.walogs.isEmpty() && verify.getMergeInfo().needsToBeChopped(tls.extent)) { log.debug("failing consistency: needs to be chopped {}", tls.extent); return false;
@Override public Collection<MergeInfo> merges() { String tableIdToModify = getConnector().tableOperations().tableIdMap().get(t3); return Collections.singletonList( new MergeInfo(new KeyExtent(tableIdToModify, null, null), MergeInfo.Operation.MERGE)); } };