/** * Gets a collection of iterator settings that should be added to the metric table for the given Accumulo table. Don't forget! Please! * * @param table Table for retrieving metrics iterators, see AccumuloClient#getTable * @return Collection of iterator settings */ public static Collection<IteratorSetting> getMetricIterators(AccumuloTable table) { String cardQualifier = new String(CARDINALITY_CQ); String rowsFamily = new String(METRICS_TABLE_ROWS_CF.array()); // Build a string for all columns where the summing combiner should be applied, // i.e. all indexed columns StringBuilder cardBuilder = new StringBuilder(rowsFamily + ":" + cardQualifier + ","); for (String s : getLocalityGroups(table).keySet()) { cardBuilder.append(s).append(":").append(cardQualifier).append(','); } cardBuilder.deleteCharAt(cardBuilder.length() - 1); // Configuration rows for the Min/Max combiners String firstRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_FIRST_ROW_CQ.array()); String lastRowColumn = rowsFamily + ":" + new String(METRICS_TABLE_LAST_ROW_CQ.array()); // Summing combiner for cardinality columns IteratorSetting s1 = new IteratorSetting(1, SummingCombiner.class, ImmutableMap.of("columns", cardBuilder.toString(), "type", "STRING")); // Min/Max combiner for the first/last rows of the table IteratorSetting s2 = new IteratorSetting(2, MinByteArrayCombiner.class, ImmutableMap.of("columns", firstRowColumn)); IteratorSetting s3 = new IteratorSetting(3, MaxByteArrayCombiner.class, ImmutableMap.of("columns", lastRowColumn)); return ImmutableList.of(s1, s2, s3); }
this.scanner.addScanIterator(new IteratorSetting(1, "firstentryiter", FirstEntryInRowIterator.class)); IteratorSetting setting = new IteratorSetting(WHOLE_ROW_ITERATOR_PRIORITY, WholeRowIterator.class); scanner.addScanIterator(setting);
IteratorSetting cfg = new IteratorSetting(100, WholeRowIterator.class); scanner.addScanIterator(cfg);
IteratorSetting cfg = new IteratorSetting(100, WholeRowIterator.class); scanner.addScanIterator(cfg);
IteratorSetting cfg = new IteratorSetting(100, WholeRowIterator.class); scanner.addScanIterator(cfg);
public ScannerOptions setColumns(SortedSet<Column> locCols) { this.fetchedColumns = locCols; // see comment in lookupTablet about why iterator is used addScanIterator(new IteratorSetting(10000, "WRI", WholeRowIterator.class.getName())); return this; } }
public TabletServerBatchDeleter(ClientContext context, Table.ID tableId, Authorizations authorizations, int numQueryThreads, BatchWriterConfig bwConfig) { super(context, tableId, authorizations, numQueryThreads); this.context = context; this.tableId = tableId; this.bwConfig = bwConfig; super.addScanIterator(new IteratorSetting(Integer.MAX_VALUE, BatchDeleter.class.getName().replaceAll("[.]", "_") + "_NOVALUE", SortedKeyIterator.class)); }
@Override public void readFields(DataInput in) throws IOException { if (in.readBoolean()) { startRow = new byte[in.readInt()]; in.readFully(startRow); } else { startRow = null; } if (in.readBoolean()) { endRow = new byte[in.readInt()]; in.readFully(endRow); } else { endRow = null; } int num = in.readInt(); iterators = new ArrayList<>(num); for (int i = 0; i < num; i++) { iterators.add(new IteratorSetting(in)); } compactionStrategy = CompactionStrategyConfigUtil.decode(in); }
@Override protected void setUpIterator(final int prio, final String name, final String term, final BatchScanner scanner, CommandLine cl, boolean negate) throws IOException { if (prio < 0) { throw new IllegalArgumentException("Priority < 0 " + prio); } final IteratorSetting si = new IteratorSetting(prio, name, RegExFilter.class); RegExFilter.setRegexs(si, term, term, term, term, true, cl.hasOption(matchSubstringOption.getOpt())); RegExFilter.setNegate(si, negate); scanner.addScanIterator(si); }
@Override protected void setTableProperties(final CommandLine cl, final Shell shellState, final int priority, final Map<String,String> options, final String classname, final String name) throws AccumuloException, AccumuloSecurityException, ShellCommandException, TableNotFoundException { // instead of setting table properties, just put the options in a list to use at scan time String profile = cl.getOptionValue(profileOpt.getOpt()); // instead of setting table properties, just put the options in a list to use at scan time for (Iterator<Entry<String,String>> i = options.entrySet().iterator(); i.hasNext();) { final Entry<String,String> entry = i.next(); if (entry.getValue() == null || entry.getValue().isEmpty()) { i.remove(); } } List<IteratorSetting> tableScanIterators = shellState.iteratorProfiles.get(profile); if (tableScanIterators == null) { tableScanIterators = new ArrayList<>(); shellState.iteratorProfiles.put(profile, tableScanIterators); } final IteratorSetting setting = new IteratorSetting(priority, name, classname); setting.addOptions(options); Iterator<IteratorSetting> iter = tableScanIterators.iterator(); while (iter.hasNext()) { if (iter.next().getName().equals(name)) { iter.remove(); } } tableScanIterators.add(setting); }
@Override public IteratorSetting getIteratorSetting(String tableName, String name, IteratorScope scope) throws AccumuloException, TableNotFoundException { checkArgument(tableName != null, "tableName is null"); checkArgument(name != null, "name is null"); checkArgument(scope != null, "scope is null"); int priority = -1; String classname = null; Map<String,String> settings = new HashMap<>(); String root = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase(), name); String opt = root + ".opt."; for (Entry<String,String> property : this.getProperties(tableName)) { if (property.getKey().equals(root)) { String parts[] = property.getValue().split(","); if (parts.length != 2) { throw new AccumuloException("Bad value for iterator setting: " + property.getValue()); } priority = Integer.parseInt(parts[0]); classname = parts[1]; } else if (property.getKey().startsWith(opt)) { settings.put(property.getKey().substring(opt.length()), property.getValue()); } } if (priority <= 0 || classname == null) { return null; } return new IteratorSetting(priority, name, classname, settings); }
protected void setTableProperties(final CommandLine cl, final Shell shellState, final int priority, final Map<String,String> options, final String classname, final String name) throws AccumuloException, AccumuloSecurityException, ShellCommandException, TableNotFoundException { // remove empty values final String tableName = OptUtil.getTableOpt(cl, shellState); ScanCommand.ensureTserversCanLoadIterator(shellState, tableName, classname); for (Iterator<Entry<String,String>> i = options.entrySet().iterator(); i.hasNext();) { final Entry<String,String> entry = i.next(); if (entry.getValue() == null || entry.getValue().isEmpty()) { i.remove(); } } final EnumSet<IteratorScope> scopes = EnumSet.noneOf(IteratorScope.class); if (cl.hasOption(allScopeOpt.getOpt()) || cl.hasOption(mincScopeOpt.getOpt())) { scopes.add(IteratorScope.minc); } if (cl.hasOption(allScopeOpt.getOpt()) || cl.hasOption(majcScopeOpt.getOpt())) { scopes.add(IteratorScope.majc); } if (cl.hasOption(allScopeOpt.getOpt()) || cl.hasOption(scanScopeOpt.getOpt())) { scopes.add(IteratorScope.scan); } if (scopes.isEmpty()) { throw new IllegalArgumentException("You must select at least one scope to configure"); } final IteratorSetting setting = new IteratorSetting(priority, name, classname, options); shellState.getAccumuloClient().tableOperations().attachIterator(tableName, setting, scopes); }
protected void setUpIterator(final int prio, final String name, final String term, final BatchScanner scanner, CommandLine cl, boolean negate) throws IOException { if (prio < 0) { throw new IllegalArgumentException("Priority < 0 " + prio); } final IteratorSetting grep = new IteratorSetting(prio, name, GrepIterator.class); GrepIterator.setTerm(grep, term); GrepIterator.setNegate(grep, negate); scanner.addScanIterator(grep); }
@Override public IteratorSetting getIteratorSetting(String namespace, String name, IteratorScope scope) throws AccumuloSecurityException, AccumuloException, NamespaceNotFoundException { if (!exists(namespace)) throw new NamespaceNotFoundException(null, namespace, null); int priority = -1; String classname = null; Map<String,String> settings = new HashMap<>(); String root = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scope.name().toLowerCase(), name); String opt = root + ".opt."; for (Entry<String,String> property : this.getProperties(namespace)) { if (property.getKey().equals(root)) { String parts[] = property.getValue().split(","); if (parts.length != 2) { throw new AccumuloException("Bad value for iterator setting: " + property.getValue()); } priority = Integer.parseInt(parts[0]); classname = parts[1]; } else if (property.getKey().startsWith(opt)) { settings.put(property.getKey().substring(opt.length()), property.getValue()); } } if (priority <= 0 || classname == null) { return null; } return new IteratorSetting(priority, name, classname, settings); }
public static Text findMax(Scanner scanner, Text start, boolean is, Text end, boolean ie) throws TableNotFoundException { scanner.setBatchSize(12); IteratorSetting cfg = new IteratorSetting(Integer.MAX_VALUE, SortedKeyIterator.class); scanner.addScanIterator(cfg); if (start == null) { start = new Text(); is = true; } if (end == null) end = findInitialEnd(scanner); return _findMax(scanner, start, is, end, ie); } }
@Override public List<IteratorSetting> getIterators() { ArrayList<IteratorSetting> ret = new ArrayList<>(); for (IterInfo ii : tac.getSsiList()) { IteratorSetting settings = new IteratorSetting(ii.getPriority(), ii.getIterName(), ii.getClassName()); Map<String,String> options = tac.getSsio().get(ii.getIterName()); settings.addOptions(options); ret.add(settings); } return ret; } }
public static IteratorSetting toIteratorSetting(TIteratorSetting tis) { return new IteratorSetting(tis.getPriority(), tis.getName(), tis.getIteratorClass(), tis.getProperties()); }
public void deleteProblemReports(Table.ID table) throws Exception { if (isMeta(table)) { Iterator<ProblemReport> pri = iterator(table); while (pri.hasNext()) { pri.next().removeFromZooKeeper(context); } return; } Scanner scanner = context.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.addScanIterator(new IteratorSetting(1, "keys-only", SortedKeyIterator.class)); scanner.setRange(new Range(new Text("~err_" + table))); Mutation delMut = new Mutation(new Text("~err_" + table)); boolean hasProblems = false; for (Entry<Key,Value> entry : scanner) { hasProblems = true; delMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier()); } if (hasProblems) MetadataTableUtil.getMetadataTable(context).update(delMut); }
@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception { final String tableName = OptUtil.getTableOpt(cl, shellState); final ScanInterpreter interpeter = getInterpreter(cl, tableName, shellState); // handle first argument, if present, the authorizations list to // scan with final Authorizations auths = getAuths(cl, shellState); final Scanner scanner = shellState.getAccumuloClient().createScanner(tableName, auths); scanner.addScanIterator( new IteratorSetting(Integer.MAX_VALUE, "NOVALUE", SortedKeyIterator.class)); // handle session-specific scan iterators addScanIterators(shellState, cl, scanner, tableName); // handle remaining optional arguments scanner.setRange(getRange(cl, interpeter)); scanner.setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS); // handle columns fetchColumns(cl, scanner, interpeter); // output / delete the records final BatchWriter writer = shellState.getAccumuloClient().createBatchWriter(tableName, new BatchWriterConfig().setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS)); FormatterConfig config = new FormatterConfig(); config.setPrintTimestamps(cl.hasOption(timestampOpt.getOpt())); shellState.printLines( new DeleterFormatter(writer, scanner, config, shellState, cl.hasOption(forceOpt.getOpt())), false); return 0; }
public static void configureScanner(ScannerBase scanner, CurrentState state) { TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); scanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME); scanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME); scanner .fetchColumnFamily(TabletsSection.SuspendLocationColumn.SUSPEND_COLUMN.getColumnFamily()); scanner.fetchColumnFamily(LogColumnFamily.NAME); scanner.fetchColumnFamily(ChoppedColumnFamily.NAME); scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class)); IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class); if (state != null) { TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers()); TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables()); TabletStateChangeIterator.setMerges(tabletChange, state.merges()); TabletStateChangeIterator.setMigrations(tabletChange, state.migrationsSnapshot()); TabletStateChangeIterator.setMasterState(tabletChange, state.getMasterState()); TabletStateChangeIterator.setShuttingDown(tabletChange, state.shutdownServers()); } scanner.addScanIterator(tabletChange); }