scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(new Text(startkey), null)); IteratorSetting cfg = new IteratorSetting(100, WholeRowIterator.class); scanner.addScanIterator(cfg); scanner.fetchColumn(colFam, new Text(field));
/** * A convenience method for setting the long encoding type. * * @param is * IteratorSetting object to configure. * @param encoderClassName * name of a class specifying the encoding type. */ public static void setEncodingType(IteratorSetting is, String encoderClassName) { is.addOption(TYPE, CLASS_PREFIX + encoderClassName); } }
public static TIteratorSetting toTIteratorSetting(IteratorSetting is) { return new TIteratorSetting(is.getPriority(), is.getName(), is.getIteratorClass(), is.getOptions()); }
public void deleteProblemReports(Table.ID table) throws Exception { if (isMeta(table)) { Iterator<ProblemReport> pri = iterator(table); while (pri.hasNext()) { pri.next().removeFromZooKeeper(context); } return; } Scanner scanner = context.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.addScanIterator(new IteratorSetting(1, "keys-only", SortedKeyIterator.class)); scanner.setRange(new Range(new Text("~err_" + table))); Mutation delMut = new Mutation(new Text("~err_" + table)); boolean hasProblems = false; for (Entry<Key,Value> entry : scanner) { hasProblems = true; delMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier()); } if (hasProblems) MetadataTableUtil.getMetadataTable(context).update(delMut); }
@Test public void testIntersect() throws Exception { conn.tableOperations().create(tableName); BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig()); Mutation m = new Mutation("part1"); m.put("bar", "doc1", "value"); m.put("bar", "doc2", "value"); m.put("dog", "doc3", "value"); m.put("foo", "doc2", "value"); m.put("foo", "doc3", "value"); bw.addMutation(m); m = new Mutation("part2"); m.put("bar", "DOC1", "value"); m.put("bar", "DOC2", "value"); m.put("dog", "DOC3", "value"); m.put("foo", "DOC2", "value"); m.put("foo", "DOC3", "value"); bw.addMutation(m); bw.flush(); final ClientSideIteratorScanner csis = new ClientSideIteratorScanner( conn.createScanner(tableName, new Authorizations())); final IteratorSetting si = new IteratorSetting(10, tableName, IntersectingIterator.class); IntersectingIterator.setColumnFamilies(si, new Text[] {new Text("bar"), new Text("foo")}); csis.addScanIterator(si); checkResults(csis, resultSet3, PartialKey.ROW_COLFAM_COLQUAL); }
@Test public void testAgeoffFilter() throws Exception { String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); is = new IteratorSetting(10, AgeOffFilter.class); AgeOffFilter.setTTL(is, 1000L); c.tableOperations().attachIterator(tableName, is); sleepUninterruptibly(500, TimeUnit.MILLISECONDS); // let zookeeper updates propagate. bw = c.createBatchWriter(tableName, bwc); Mutation m = new Mutation("foo"); m.put("a", "b", "c"); bw.addMutation(m); bw.close(); sleepUninterruptibly(1, TimeUnit.SECONDS); assertEquals(0, Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator())); }
String tableName = getUniqueNames(1)[0]; Connector c = getConnector(); c.tableOperations().create(tableName); log.info("Adding slow iterator"); IteratorSetting setting = new IteratorSetting(50, SlowIterator.class); SlowIterator.setSleepTime(setting, 1000); c.tableOperations().attachIterator(tableName, setting); log.info("Splitting the table"); SortedSet<Text> partitionKeys = new TreeSet<>(); partitionKeys.add(new Text("5")); c.tableOperations().addSplits(tableName, partitionKeys); log.info("waiting for zookeeper propagation"); UtilWaitThread.sleep(5 * 1000); bw.close(); log.info("Fetching some entries: should timeout and return something"); Scanner s = c.createScanner(tableName, Authorizations.EMPTY); s.setBatchTimeout(500, TimeUnit.MILLISECONDS); testScanner(s, 1200); BatchScanner bs = c.createBatchScanner(tableName, Authorizations.EMPTY, 5); bs.setBatchTimeout(500, TimeUnit.MILLISECONDS); bs.setRanges(Collections.singletonList(new Range())); testScanner(bs, 1200);
private void test(String t1, String t2, Connector c, IteratorSetting itset, int numEntriesToWritePerEntry) throws Exception { c.tableOperations().create(t1); Key k = new Key(new Text("row"), new Text("cf"), new Text("cq")); Value v = new Value("1".getBytes()); BatchWriter writer = c.createBatchWriter(t1, config); Mutation m = new Mutation(k.getRow()); m.put(k.getColumnFamily(), k.getColumnQualifier(), v); writer.addMutation(m); writer.close(); c.tableOperations().create(t2); IteratorSetting summer = new IteratorSetting(2, "summer", SummingCombiner.class); LongCombiner.setEncodingType(summer, LongCombiner.Type.STRING); LongCombiner.setCombineAllColumns(summer, true); c.tableOperations().attachIterator(t2, summer); Scanner scanner = c.createScanner(t1, Authorizations.EMPTY); scanner.addScanIterator(itset); actual = Iterators.getOnlyElement(scanner.iterator()); Assert.assertTrue(actual.getKey().equals(k, PartialKey.ROW_COLFAM_COLQUAL)); Assert.assertEquals(BatchWriterIterator.SUCCESS_VALUE, actual.getValue()); scanner.close();
@Test public void testCompactEmptyTableWithGeneratorIterator_Splits() throws TableExistsException, AccumuloException, AccumuloSecurityException, TableNotFoundException { String tableName = getUniqueNames(1)[0]; connector.tableOperations().create(tableName); SortedSet<Text> splitset = new TreeSet<>(); splitset.add(new Text("f")); connector.tableOperations().addSplits(tableName, splitset); List<IteratorSetting> list = new ArrayList<>(); list.add(new IteratorSetting(15, HardListIterator.class)); connector.tableOperations().compact(tableName, null, null, list, true, true); Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY); Map<Key,Value> actual = new TreeMap<>(COMPARE_KEY_TO_COLQ); // only compare row, colF, colQ for (Map.Entry<Key,Value> entry : scanner) actual.put(entry.getKey(), entry.getValue()); assertEquals(HardListIterator.allEntriesToInject, actual); connector.tableOperations().delete(tableName); }
this.scanner.addScanIterator(new IteratorSetting(1, "firstentryiter", FirstEntryInRowIterator.class)); Text family = new Text(); Text qualifier = new Text(); family.set(columnHandle.getFamily().get()); qualifier.set(columnHandle.getQualifier().get()); this.scanner.fetchColumn(family, qualifier); IteratorSetting setting = new IteratorSetting(WHOLE_ROW_ITERATOR_PRIORITY, WholeRowIterator.class); scanner.addScanIterator(setting);
final Connector conn = getConnector(); final String tableName = getUniqueNames(1)[0]; conn.tableOperations().create(tableName); try (BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig())) { Mutation m = new Mutation("row1"); m.put("bob", "2", EMPTY); m.put("frank", "3", EMPTY); m.put("steve", "1", EMPTY); bw.addMutation(m); m = new Mutation("row2"); m.put("mort", "6", EMPTY); m.put("zed", "5", EMPTY); bw.addMutation(m); IteratorSetting is = new IteratorSetting(50, OrIterator.class); is.addOption(OrIterator.COLUMNS_KEY, "mort,frank"); Map<String,String> expectedData = new HashMap<>(); expectedData.put("frank", "3"); expectedData.put("mort", "6");
@Test public void testBugFix() throws Exception { // test bug fox for ACCUMULO-3977 String table = super.getUniqueNames(1)[0]; Connector conn = getConnector(); conn.tableOperations().create(table); BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); addDocument(bw, "000A", "dog", "cat", "hamster", "iguana", "the"); addDocument(bw, "000B", "java", "perl", "C++", "pascal", "the"); addDocument(bw, "000C", "chrome", "firefox", "safari", "opera", "the"); addDocument(bw, "000D", "logarithmic", "quadratic", "linear", "exponential", "the"); bw.close(); // its a bug when using rfiles, so flush conn.tableOperations().flush(table, null, null, true); IteratorSetting iterCfg = new IteratorSetting(30, "ayeaye", IntersectingIterator.class.getName()); IntersectingIterator.setColumnFamilies(iterCfg, new Text[] {new Text("the"), new Text("hamster")}); Scanner scanner = conn.createScanner(table, Authorizations.EMPTY); scanner.enableIsolation(); scanner.addScanIterator(iterCfg); for (int i = 0; i < 100; i++) { Iterator<Entry<Key,Value>> iter = scanner.iterator(); Assert.assertTrue(iter.hasNext()); Assert.assertEquals("000A", iter.next().getKey().getColumnQualifierData().toString()); Assert.assertFalse(iter.hasNext()); } }
public static Text findMax(Scanner scanner, Text start, boolean is, Text end, boolean ie) throws TableNotFoundException { scanner.setBatchSize(12); IteratorSetting cfg = new IteratorSetting(Integer.MAX_VALUE, SortedKeyIterator.class); scanner.addScanIterator(cfg); if (start == null) { start = new Text(); is = true; } if (end == null) end = findInitialEnd(scanner); return _findMax(scanner, start, is, end, ie); } }
IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class); bs.addScanIterator(cfg); WorkSection.limit(bs); bs.setRanges(Collections.singleton(new Range())); Text buffer = new Text(); serializedRow.getKey().getRow(), entry.getKey(), entry.getValue()); Mutation replMutation = new Mutation(buffer); replBw.addMutation(replMutation); } catch (MutationsRejectedException e) { log.error("Error writing mutations to update replication Status" replBw.close(); } catch (MutationsRejectedException e) { log.error("Error writing mutations to update replication Status"
iterators = new ArrayList<>(); for (int i = 0; i < iterSize; i++) iterators.add(new IteratorSetting(dataInput)); ranges = new ArrayList<>(); for (int i = 0; i < rangeSize; i++) { Range range = new Range(); range.readFields(dataInput); ranges.add(range); for (int i = 0; i < columnSize; i++) { long numPairs = dataInput.readInt(); Text colFam = new Text(); colFam.readFields(dataInput); if (numPairs == 1) { columns.add(new Pair<>(colFam, null)); } else if (numPairs == 2) { Text colQual = new Text(); colQual.readFields(dataInput); columns.add(new Pair<>(colFam, colQual));
Range allTables = MetadataSchema.TabletsSection.getRange(); Range tableRange = MetadataSchema.TabletsSection.getRange(tableId); Range beforeTable = new Range(allTables.getStartKey(), true, tableRange.getStartKey(), false); Range afterTable = new Range(tableRange.getEndKey(), false, allTables.getEndKey(), true); bs.setRanges(Arrays.asList(beforeTable, afterTable)); bs.fetchColumnFamily(DataFileColumnFamily.NAME); IteratorSetting cfg = new IteratorSetting(40, "grep", GrepIterator.class); GrepIterator.setTerm(cfg, "/" + tableId + "/"); bs.addScanIterator(cfg); if (entry.getKey().getColumnQualifier().toString().contains("/" + tableId + "/")) { refCount++;
protected List<CombinerConfiguration> createWorkingCombinerConfigurations() { List<CombinerConfiguration> lag = new ArrayList<>(); for (int counter = 0; counter < 3; counter++) { String colFamilyName = String.format("column-family-%d", counter); lag.add(new CombinerConfiguration(new Column(new Text(colFamilyName)), new IteratorSetting(10, "agg", aggClassName))); } return lag; }
public static void setQueryInfo(Job job, Set<String> entityTypes, EntityShardBuilder shardBuilder, TypeRegistry<String> typeRegistry) throws AccumuloSecurityException, AccumuloException, TableNotFoundException, IOException { validateOptions(job); Collection<Range> ranges = new LinkedList<Range>(); for (String type : entityTypes) { Set<Text> shards = shardBuilder.buildShardsForTypes(singleton(type)); for (Text shard : shards) ranges.add(prefix(shard.toString(), type)); } setRanges(job, ranges); IteratorSetting iteratorSetting = new IteratorSetting(16, "wholeColumnFamilyIterator", WholeColumnFamilyIterator.class); addIterator(job, iteratorSetting); job.getConfiguration().setBoolean(QUERY, false); job.getConfiguration().set(TYPE_REGISTRY, new String(toBase64(typeRegistry))); }
final Connector conn = getConnector(); final String tableName = getUniqueNames(1)[0]; conn.tableOperations().create(tableName); try (BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig())) { Mutation m = new Mutation("row1"); m.put("bob", "2", EMPTY); m.put("frank", "3", EMPTY); m.put("steve", "1", EMPTY); bw.addMutation(m); IteratorSetting is = new IteratorSetting(50, OrIterator.class); is.addOption(OrIterator.COLUMNS_KEY, "bob,steve"); try (Scanner s = conn.createScanner(tableName, Authorizations.EMPTY)) { s.addScanIterator(is); Iterator<Entry<Key,Value>> iter = s.iterator(); assertTrue(iter.hasNext()); Key k = iter.next().getKey();
private void runTest(Authorizations auths, boolean shouldFail) throws Exception { ClusterUser clusterUser = getUser(0); Connector userC = getCluster().getConnector(clusterUser.getPrincipal(), clusterUser.getToken()); writeTestMutation(userC); IteratorSetting setting = new IteratorSetting(10, AuthsIterator.class); Scanner scanner = userC.createScanner(tableName, auths); scanner.addScanIterator(setting); BatchScanner batchScanner = userC.createBatchScanner(tableName, auths, 1); batchScanner.setRanges(Collections.singleton(new Range("1"))); batchScanner.addScanIterator(setting); runTest(scanner, auths, shouldFail); runTest(batchScanner, auths, shouldFail); scanner.close(); batchScanner.close(); }