public void setIterator(String table, IteratorSetting setting) { try { // Remove any existing iterator settings of the same name, if applicable Map<String, EnumSet<IteratorScope>> iterators = connector.tableOperations().listIterators(table); if (iterators.containsKey(setting.getName())) { connector.tableOperations().removeIterator(table, setting.getName(), iterators.get(setting.getName())); } connector.tableOperations().attachIterator(table, setting); } catch (AccumuloSecurityException | AccumuloException e) { throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to set iterator on table " + table, e); } catch (TableNotFoundException e) { throw new PrestoException(ACCUMULO_TABLE_DNE, "Failed to set iterator, table does not exist", e); } }
conn.tableOperations().attachIterator(table.getMetricsTableName(), s);
conn.tableOperations().attachIterator(table.getMetricsTableName(), s);
protected void setTableProperties(final CommandLine cl, final Shell shellState, final int priority, final Map<String,String> options, final String classname, final String name) throws AccumuloException, AccumuloSecurityException, ShellCommandException, TableNotFoundException { // remove empty values final String tableName = OptUtil.getTableOpt(cl, shellState); ScanCommand.ensureTserversCanLoadIterator(shellState, tableName, classname); for (Iterator<Entry<String,String>> i = options.entrySet().iterator(); i.hasNext();) { final Entry<String,String> entry = i.next(); if (entry.getValue() == null || entry.getValue().isEmpty()) { i.remove(); } } final EnumSet<IteratorScope> scopes = EnumSet.noneOf(IteratorScope.class); if (cl.hasOption(allScopeOpt.getOpt()) || cl.hasOption(mincScopeOpt.getOpt())) { scopes.add(IteratorScope.minc); } if (cl.hasOption(allScopeOpt.getOpt()) || cl.hasOption(majcScopeOpt.getOpt())) { scopes.add(IteratorScope.majc); } if (cl.hasOption(allScopeOpt.getOpt()) || cl.hasOption(scanScopeOpt.getOpt())) { scopes.add(IteratorScope.scan); } if (scopes.isEmpty()) { throw new IllegalArgumentException("You must select at least one scope to configure"); } final IteratorSetting setting = new IteratorSetting(priority, name, classname, options); shellState.getAccumuloClient().tableOperations().attachIterator(tableName, setting, scopes); }
Collections.singletonList(new Column(MetadataSchema.ReplicationSection.COLF))); try { tops.attachIterator(tableName, setting); } catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) { throw new RuntimeException(e);
IteratorSetting setting = new IteratorSetting(10, "ageoff", AgeOffFilter.class.getName()); AgeOffFilter.setTTL(setting, 7 * 24 * 60 * 60 * 1000L); accumuloClient.tableOperations().attachIterator(tableName, setting);
@Override public void configure(TableOperations tops) { if (null != this.ageoff) { EnumSet<IteratorScope> scopes = EnumSet.of(IteratorScope.scan, IteratorScope.minc, IteratorScope.majc); HashMap<String,String> properties = new HashMap<>(); properties.put("ttl", ageoff); IteratorSetting settings = new IteratorSetting(19, AgeOffFilter.class, properties); try { tops.attachIterator(tableName, settings, scopes); } catch (IllegalArgumentException | AccumuloException iaEx) { String msg = iaEx.getMessage(); if (msg.contains("name conflict")) log.info("Iterator, 'age-off' already exists for table: " + tableName + "\n" + iaEx.getMessage()); else log.error("Error setting up age-off iterator on table: " + tableName, iaEx); } catch (Exception e) { log.error("Error setting up age-off iterator on table: " + tableName, e); } } }
public void addIterator(String tableName, IteratorSetting iterSet) throws D4mException { TableOperations tops = this.connector.tableOperations(); try { tops.attachIterator(tableName, iterSet); // adds on all scopes: majc, minc, scan } catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) { log.warn("",e); throw new D4mException(e); } }
private void applyAgeOffIterator(Connector con, String tableName, boolean useIterator) throws Exception { IteratorSetting ageOffIteratorSettings = null; if (useIterator) { ageOffIteratorSettings = new IteratorSetting(100, "ageoff", MetricAgeOffIterator.class, this.ageOff); } else { ageOffIteratorSettings = new IteratorSetting(100, "ageoff", MetricAgeOffFilter.class, this.ageOff); } connector.tableOperations().attachIterator(tableName, ageOffIteratorSettings, AGEOFF_SCOPES); }
@Override public void configureTable(String table, TableOperations tops, Configuration conf) throws Exception { IteratorSetting is = tops.getIteratorSetting(table, ITER_NAME, IteratorUtil.IteratorScope.scan); String metricName = metric.toString(); if (is == null) { // create a fresh iterator Map<String,String> options = new TreeMap<>(); options.put("type", "STRING"); options.put("columns", metricName); is = new IteratorSetting(ITER_PRIORITY, ITER_NAME, SummingCombiner.class, options); tops.attachIterator(table, is); } else { // if iterator exists, piggyback on it String columns = is.getOptions().get("columns"); if (!columns.contains(metricName)) { for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) { String config = String.format("table.iterator.%s.%s.opt.columns", scope, ITER_NAME); tops.setProperty(table, config, columns.concat("," + metricName)); } } } }
protected static void ensureRowDeletingIteratorIsAttached(Connector connector, String tableName) { try { synchronized (addIteratorLock) { IteratorSetting is = new IteratorSetting(ROW_DELETING_ITERATOR_PRIORITY, ROW_DELETING_ITERATOR_NAME, RowDeletingIterator.class); if (!connector.tableOperations().listIterators(tableName).containsKey(ROW_DELETING_ITERATOR_NAME)) { try { connector.tableOperations().attachIterator(tableName, is); } catch (Exception ex) { // If many processes are starting up at the same time (see YARN). It's possible that there will be a collision. final int SLEEP_TIME = 5000; LOGGER.warn("Failed to attach RowDeletingIterator. Retrying in " + SLEEP_TIME + "ms."); Thread.sleep(SLEEP_TIME); if (!connector.tableOperations().listIterators(tableName).containsKey(ROW_DELETING_ITERATOR_NAME)) { connector.tableOperations().attachIterator(tableName, is); } } } } } catch (Exception e) { throw new VertexiumException("Could not attach RowDeletingIterator", e); } }
protected static void ensureRowDeletingIteratorIsAttached(Connector connector, String tableName) { try { synchronized (addIteratorLock) { IteratorSetting is = new IteratorSetting(ROW_DELETING_ITERATOR_PRIORITY, ROW_DELETING_ITERATOR_NAME, RowDeletingIterator.class); if (!connector.tableOperations().listIterators(tableName).containsKey(ROW_DELETING_ITERATOR_NAME)) { try { connector.tableOperations().attachIterator(tableName, is); } catch (Exception ex) { // If many processes are starting up at the same time (see YARN). It's possible that there will be a collision. final int SLEEP_TIME = 5000; LOGGER.warn("Failed to attach RowDeletingIterator. Retrying in %dms.", SLEEP_TIME); Thread.sleep(SLEEP_TIME); if (!connector.tableOperations().listIterators(tableName).containsKey(ROW_DELETING_ITERATOR_NAME)) { connector.tableOperations().attachIterator(tableName, is); } } } } } catch (Exception e) { throw new VertexiumException("Could not attach RowDeletingIterator", e); } }
protected static void ensureRowDeletingIteratorIsAttached(Connector connector, String tableName) { try { synchronized (addIteratorLock) { IteratorSetting is = new IteratorSetting(ROW_DELETING_ITERATOR_PRIORITY, ROW_DELETING_ITERATOR_NAME, RowDeletingIterator.class); if (!connector.tableOperations().listIterators(tableName).containsKey(ROW_DELETING_ITERATOR_NAME)) { try { connector.tableOperations().attachIterator(tableName, is); } catch (Exception ex) { // If many processes are starting up at the same time (see YARN). It's possible that there will be a collision. final int SLEEP_TIME = 5000; LOGGER.warn("Failed to attach RowDeletingIterator. Retrying in %dms.", SLEEP_TIME); Thread.sleep(SLEEP_TIME); if (!connector.tableOperations().listIterators(tableName).containsKey(ROW_DELETING_ITERATOR_NAME)) { connector.tableOperations().attachIterator(tableName, is); } } } } } catch (Exception e) { throw new VertexiumException("Could not attach RowDeletingIterator", e); } }
private void tableCheck(Connector c) throws AccumuloException, AccumuloSecurityException, TableExistsException { if (!c.tableOperations().exists(TABLE_NAME)) { c.tableOperations().create(TABLE_NAME); try { IteratorSetting iteratorCfg = new IteratorSetting(19, "ageoff", QueriesTableAgeOffIterator.class); c.tableOperations().attachIterator(TABLE_NAME, iteratorCfg, EnumSet.allOf(IteratorScope.class)); } catch (TableNotFoundException e) { throw new AccumuloException("We just created " + TABLE_NAME + " so this shouldn't have happened!", e); } } }
@Override public void attachIterator(ByteBuffer login, String tableName, org.apache.accumulo.proxy.thrift.IteratorSetting setting, Set<org.apache.accumulo.proxy.thrift.IteratorScope> scopes) throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.TableNotFoundException, TException { try { getConnector(login).tableOperations().attachIterator(tableName, getIteratorSetting(setting), getIteratorScopes(scopes)); } catch (Exception e) { handleExceptionTNF(e); } }
/** * This method can be used to attach an iterator to the table in use by the * store instance. * * @param store the accumulo store * @param iteratorSetting the iterator setting to add. * @throws StoreException if any issues occur adding an iterator setting */ public static void addIterator(final AccumuloStore store, final IteratorSetting iteratorSetting) throws StoreException { if (null != iteratorSetting) { try { store.getConnection().tableOperations().attachIterator(store.getTableName(), iteratorSetting); } catch (final AccumuloSecurityException | AccumuloException | TableNotFoundException e) { throw new StoreException("Add iterator with Name: " + iteratorSetting.getName(), e); } } TableUtils.setLocalityGroups(store); }
@Test public void aggregationTest() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class); SummingCombiner.setEncodingType(setting, Type.STRING); SummingCombiner.setColumns(setting, Collections.singletonList(new IteratorSetting.Column("cf"))); c.tableOperations().attachIterator(tableName, setting); BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); for (int i = 0; i < 10; i++) { Mutation m = new Mutation("row1"); m.put("cf".getBytes(), "col1".getBytes(), ("" + i).getBytes()); bw.addMutation(m); } bw.close(); checkSum(tableName, c); }
SummingCombiner.class.getName(), options); tops.attachIterator(table, is);
@Test public void test() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); IteratorSetting setting = new IteratorSetting(100, SlowIterator.class); SlowIterator.setSleepTime(setting, 1000); c.tableOperations().attachIterator(tableName, setting, EnumSet.of(IteratorScope.minc)); // let the configuration change propagate through zookeeper UtilWaitThread.sleep(1000); Mutation m = new Mutation("xyzzy"); for (int i = 0; i < 100; i++) { m.put("cf", "" + i, new Value(new byte[] {})); } BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); bw.addMutation(m); bw.close(); // should go fast c.tableOperations().delete(tableName); }
@Test public void testAgeoffFilter() throws Exception { String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); is = new IteratorSetting(10, AgeOffFilter.class); AgeOffFilter.setTTL(is, 1000L); c.tableOperations().attachIterator(tableName, is); sleepUninterruptibly(500, TimeUnit.MILLISECONDS); // let zookeeper updates propagate. bw = c.createBatchWriter(tableName, bwc); Mutation m = new Mutation("foo"); m.put("a", "b", "c"); bw.addMutation(m); bw.close(); sleepUninterruptibly(1, TimeUnit.SECONDS); assertEquals(0, Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator())); }