return state; if (total == 0) { log.trace("failed to see any tablets for this range, ignoring {}", info.getExtent()); return state; info.getExtent(), state, info.isDelete()); if (state == MergeState.STARTED) { state = MergeState.SPLITTING; log.info("{} are hosted, total {}", hosted, total); if (!info.isDelete() && total == 1) { log.info("Merge range is already contained in a single tablet {}", info.getExtent()); state = MergeState.COMPLETE; } else if (hosted == total) { if (info.isDelete()) { if (!lowerSplit) log.info("Waiting for {} lower split to occur {}", info, info.getExtent()); else if (!upperSplit) log.info("Waiting for {} upper split to occur {}", info, info.getExtent()); else state = MergeState.WAITING_FOR_CHOPPED; log.info("Waiting for {} hosted tablets to be {} {}", hosted, total, info.getExtent()); log.info("{} tablets are chopped {}", chopped, info.getExtent()); if (chopped == needsToBeChopped) { state = MergeState.WAITING_FOR_OFFLINE; } else { log.info("Waiting for {} chopped tablets to be {} {}", chopped, needsToBeChopped,
public static void setMerges(IteratorSetting cfg, Collection<MergeInfo> merges) { DataOutputBuffer buffer = new DataOutputBuffer(); try { for (MergeInfo info : merges) { KeyExtent extent = info.getExtent(); if (extent != null && !info.getState().equals(MergeState.NONE)) { info.write(buffer); } } } catch (Exception ex) { throw new RuntimeException(ex); } String encoded = Base64.getEncoder() .encodeToString(Arrays.copyOf(buffer.getData(), buffer.getLength())); cfg.addOption(MERGES_OPTION, encoded); }
public void update(KeyExtent ke, TabletState state, boolean chopped, boolean hasWALs) { if (info.getState().equals(MergeState.NONE)) return; if (!upperSplit && info.getExtent().getEndRow().equals(ke.getPrevEndRow())) { log.info("Upper split found"); upperSplit = true; } if (!lowerSplit && info.getExtent().getPrevEndRow().equals(ke.getEndRow())) { log.info("Lower split found"); lowerSplit = true; } if (!info.overlaps(ke)) return; if (info.needsToBeChopped(ke)) { this.needsToBeChopped++; if (chopped) { if (state.equals(TabletState.HOSTED)) { this.chopped++; } else if (!hasWALs) { this.chopped++; } } } this.total++; if (state.equals(TabletState.HOSTED)) this.hosted++; if (state.equals(TabletState.UNASSIGNED) || state.equals(TabletState.SUSPENDED)) this.unassigned++; }
public static void main(String[] args) throws Exception { ServerUtilOpts opts = new ServerUtilOpts(); opts.parseArgs(MergeStats.class.getName(), args); try (AccumuloClient client = opts.createClient()) { Map<String,String> tableIdMap = client.tableOperations().tableIdMap(); ZooReaderWriter zooReaderWriter = opts.getServerContext().getZooReaderWriter(); for (Entry<String,String> entry : tableIdMap.entrySet()) { final String table = entry.getKey(), tableId = entry.getValue(); String path = ZooUtil.getRoot(client.getInstanceID()) + Constants.ZTABLES + "/" + tableId + "/merge"; MergeInfo info = new MergeInfo(); if (zooReaderWriter.exists(path)) { byte[] data = zooReaderWriter.getData(path, new Stat()); DataInputBuffer in = new DataInputBuffer(); in.reset(data, data.length); info.readFields(in); } System.out.println(String.format("%25s %10s %10s %s", table, info.getState(), info.getOperation(), info.getExtent())); } } } }
private void deleteTablets(MergeInfo info, Range scanRange, BatchWriter bw, AccumuloClient client) throws TableNotFoundException, MutationsRejectedException { Scanner scanner; Mutation m; // Delete everything in the other tablets // group all deletes into tablet into one mutation, this makes tablets // either disappear entirely or not all.. this is important for the case // where the process terminates in the loop below... scanner = client.createScanner(info.getExtent().isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY); Master.log.debug("Deleting range {}", scanRange); scanner.setRange(scanRange); RowIterator rowIter = new RowIterator(scanner); while (rowIter.hasNext()) { Iterator<Entry<Key,Value>> row = rowIter.next(); m = null; while (row.hasNext()) { Entry<Key,Value> entry = row.next(); Key key = entry.getKey(); if (m == null) m = new Mutation(key.getRow()); m.putDelete(key.getColumnFamily(), key.getColumnQualifier()); Master.log.debug("deleting entry {}", key); } bw.addMutation(m); } bw.flush(); }
return; KeyExtent range = info.getExtent(); if (tls.extent.overlaps(range)) { for (Text splitPoint : new Text[] {range.getPrevEndRow(), range.getEndRow()}) {
public void setMergeState(MergeInfo info, MergeState state) throws KeeperException, InterruptedException { synchronized (mergeLock) { String path = getZooKeeperRoot() + Constants.ZTABLES + "/" + info.getExtent().getTableId() + "/merge"; info.setState(state); if (state.equals(MergeState.NONE)) { context.getZooReaderWriter().recursiveDelete(path, NodeMissingPolicy.SKIP); } else { DataOutputBuffer out = new DataOutputBuffer(); try { info.write(out); } catch (IOException ex) { throw new RuntimeException("Unlikely", ex); } context.getZooReaderWriter().putPersistentData(path, out.getData(), state.equals(MergeState.STARTED) ? ZooUtil.NodeExistsPolicy.FAIL : ZooUtil.NodeExistsPolicy.OVERWRITE); } mergeLock.notifyAll(); } nextEvent.event("Merge state of %s set to %s", info.getExtent(), state); }
if (mergeInfo.getExtent() != null) { log.debug("mergeInfo overlaps: {} {}", extent, mergeInfo.overlaps(extent)); if (mergeInfo.overlaps(extent)) {
private boolean verifyMergeConsistency(AccumuloClient accumuloClient, CurrentState master) throws TableNotFoundException, IOException { MergeStats verify = new MergeStats(info); KeyExtent extent = info.getExtent(); Scanner scanner = accumuloClient .createScanner(extent.isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY);
"Unable to update merge state for merge " + stats.getMergeInfo().getExtent(), ex);
private void deleteTablets(MergeInfo info) throws AccumuloException { KeyExtent extent = info.getExtent(); String targetSystemTable = extent.isMeta() ? RootTable.NAME : MetadataTable.NAME; Master.log.debug("Deleting tablets for {}", extent);
private void mergeMetadataRecords(MergeInfo info) throws AccumuloException { KeyExtent range = info.getExtent(); Master.log.debug("Merging metadata for {}", range); KeyExtent stop = getHighTablet(range);
public static void setMerges(IteratorSetting cfg, Collection<MergeInfo> merges) { DataOutputBuffer buffer = new DataOutputBuffer(); try { for (MergeInfo info : merges) { KeyExtent extent = info.getExtent(); if (extent != null && !info.getState().equals(MergeState.NONE)) { info.write(buffer); } } } catch (Exception ex) { throw new RuntimeException(ex); } String encoded = Base64.encodeBase64String(Arrays.copyOf(buffer.getData(), buffer.getLength())); cfg.addOption(MERGES_OPTION, encoded); }
public void update(KeyExtent ke, TabletState state, boolean chopped, boolean hasWALs) { if (info.getState().equals(MergeState.NONE)) return; if (!upperSplit && info.getExtent().getEndRow().equals(ke.getPrevEndRow())) { log.info("Upper split found"); upperSplit = true; } if (!lowerSplit && info.getExtent().getPrevEndRow().equals(ke.getEndRow())) { log.info("Lower split found"); lowerSplit = true; } if (!info.overlaps(ke)) return; if (info.needsToBeChopped(ke)) { this.needsToBeChopped++; if (chopped) { if (state.equals(TabletState.HOSTED)) { this.chopped++; } else if (!hasWALs) { this.chopped++; } } } this.total++; if (state.equals(TabletState.HOSTED)) this.hosted++; if (state.equals(TabletState.UNASSIGNED) || state.equals(TabletState.SUSPENDED)) this.unassigned++; }
public static void main(String[] args) throws Exception { ClientOpts opts = new ClientOpts(); opts.parseArgs(MergeStats.class.getName(), args); Connector conn = opts.getConnector(); Map<String,String> tableIdMap = conn.tableOperations().tableIdMap(); for (Entry<String,String> entry : tableIdMap.entrySet()) { final String table = entry.getKey(), tableId = entry.getValue(); String path = ZooUtil.getRoot(conn.getInstance().getInstanceID()) + Constants.ZTABLES + "/" + tableId + "/merge"; MergeInfo info = new MergeInfo(); if (ZooReaderWriter.getInstance().exists(path)) { byte[] data = ZooReaderWriter.getInstance().getData(path, new Stat()); DataInputBuffer in = new DataInputBuffer(); in.reset(data, data.length); info.readFields(in); } System.out.println(String.format("%25s %10s %10s %s", table, info.getState(), info.getOperation(), info.getExtent())); } } }
private void deleteTablets(MergeInfo info, Range scanRange, BatchWriter bw, Connector conn) throws TableNotFoundException, MutationsRejectedException { Scanner scanner; Mutation m; // Delete everything in the other tablets // group all deletes into tablet into one mutation, this makes tablets // either disappear entirely or not all.. this is important for the case // where the process terminates in the loop below... scanner = conn.createScanner(info.getExtent().isMeta() ? RootTable.NAME : MetadataTable.NAME, Authorizations.EMPTY); Master.log.debug("Deleting range " + scanRange); scanner.setRange(scanRange); RowIterator rowIter = new RowIterator(scanner); while (rowIter.hasNext()) { Iterator<Entry<Key,Value>> row = rowIter.next(); m = null; while (row.hasNext()) { Entry<Key,Value> entry = row.next(); Key key = entry.getKey(); if (m == null) m = new Mutation(key.getRow()); m.putDelete(key.getColumnFamily(), key.getColumnQualifier()); Master.log.debug("deleting entry " + key); } bw.addMutation(m); } bw.flush(); }
public void setMergeState(MergeInfo info, MergeState state) throws IOException, KeeperException, InterruptedException { synchronized (mergeLock) { String path = ZooUtil.getRoot(getInstance().getInstanceID()) + Constants.ZTABLES + "/" + info.getExtent().getTableId() + "/merge"; info.setState(state); if (state.equals(MergeState.NONE)) { ZooReaderWriter.getInstance().recursiveDelete(path, NodeMissingPolicy.SKIP); } else { DataOutputBuffer out = new DataOutputBuffer(); try { info.write(out); } catch (IOException ex) { throw new RuntimeException("Unlikely", ex); } ZooReaderWriter.getInstance().putPersistentData(path, out.getData(), state.equals(MergeState.STARTED) ? ZooUtil.NodeExistsPolicy.FAIL : ZooUtil.NodeExistsPolicy.OVERWRITE); } mergeLock.notifyAll(); } nextEvent.event("Merge state of %s set to %s", info.getExtent(), state); }