@Override public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException, AccumuloException { compact(tableName, new CompactionConfig().setStartRow(start).setEndRow(end) .setIterators(iterators).setFlush(flush).setWait(wait)); }
for (IteratorSetting setting : config.getIterators()) { String iteratorClass = setting.getIteratorClass(); if (!testClassLoad(tableName, iteratorClass, skviName)) { final String compactionStrategyName = config.getCompactionStrategy().getClassName(); if (!CompactionStrategyConfigUtil.DEFAULT_STRATEGY.getClassName() .equals(compactionStrategyName)) { Text start = config.getStartRow(); Text end = config.getEndRow(); if (config.getFlush()) _flush(tableId, start, end, true); start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(config.getIterators())), ByteBuffer.wrap(CompactionStrategyConfigUtil.encode(config.getCompactionStrategy()))); doFateOperation(FateOperation.TABLE_COMPACT, args, opts, tableName, config.getWait()); } catch (TableExistsException | NamespaceExistsException e) {
@Override protected void doTableOp(final Shell shellState, final String tableName) throws AccumuloException, AccumuloSecurityException { // compact the tables if (cancel) { try { shellState.getAccumuloClient().tableOperations().cancelCompaction(tableName); Shell.log.info("Compaction canceled for table " + tableName); } catch (TableNotFoundException e) { throw new AccumuloException(e); } } else { try { if (compactionConfig.getWait()) { Shell.log.info("Compacting table ..."); } for (IteratorSetting iteratorSetting : compactionConfig.getIterators()) { ScanCommand.ensureTserversCanLoadIterator(shellState, tableName, iteratorSetting.getIteratorClass()); } shellState.getAccumuloClient().tableOperations().compact(tableName, compactionConfig); Shell.log.info("Compaction of table " + tableName + " " + (compactionConfig.getWait() ? "completed" : "started") + " for given range"); } catch (Exception ex) { throw new AccumuloException(ex); } } }
/** * Creates a table, adds a record to it, and then compacts the table. A simple way to make sure * that the system user exists (since the master does an RPC to the tserver which will create the * system user if it doesn't already exist). */ private void createTableWithDataAndCompact(Connector conn) throws TableNotFoundException, AccumuloSecurityException, AccumuloException, TableExistsException { final String table = testName.getMethodName() + "_table"; conn.tableOperations().create(table); BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); Mutation m = new Mutation("a"); m.put("b", "c", "d"); bw.addMutation(m); bw.close(); conn.tableOperations().compact(table, new CompactionConfig().setFlush(true).setWait(true)); } }
private void testDropNone(Map<String,String> options) throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); writeFlush(c, tableName, "a"); writeFlush(c, tableName, "b"); CompactionStrategyConfig csConfig = new CompactionStrategyConfig( TestCompactionStrategy.class.getName()); csConfig.setOptions(options); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig)); Assert.assertEquals(ImmutableSet.of("a", "b"), getRows(c, tableName)); }
@Test public void testIterators() throws Exception { // test compaction strategy + iterators Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); writeFlush(c, tableName, "a"); writeFlush(c, tableName, "b"); // create a file that starts with A containing rows 'a' and 'b' c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); writeFlush(c, tableName, "c"); writeFlush(c, tableName, "d"); Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName)); // drop files that start with A CompactionStrategyConfig csConfig = new CompactionStrategyConfig( TestCompactionStrategy.class.getName()); csConfig.setOptions(ImmutableMap.of("inputPrefix", "F")); IteratorSetting iterConf = new IteratorSetting(21, "myregex", RegExFilter.class); RegExFilter.setRegexs(iterConf, "a|c", null, null, null, false); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true) .setCompactionStrategy(csConfig).setIterators(Arrays.asList(iterConf))); // compaction strategy should only be applied to one file. If its applied to both, then row 'b' // would be dropped by filter. Assert.assertEquals(ImmutableSet.of("a", "b", "c"), getRows(c, tableName)); Assert.assertEquals(2, FunctionalTestUtils.countRFiles(c, tableName)); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); // ensure that iterator is not applied Assert.assertEquals(ImmutableSet.of("a", "b", "c"), getRows(c, tableName)); Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName)); }
@Test public void testConcurrent() throws Exception { // two compactions without iterators or strategy should be able to run concurrently Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); // write random data because its very unlikely it will compress writeRandomValue(c, tableName, 1 << 16); writeRandomValue(c, tableName, 1 << 16); c.tableOperations().compact(tableName, new CompactionConfig().setWait(false)); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName)); writeRandomValue(c, tableName, 1 << 16); IteratorSetting iterConfig = new IteratorSetting(30, SlowIterator.class); SlowIterator.setSleepTime(iterConfig, 1000); long t1 = System.currentTimeMillis(); c.tableOperations().compact(tableName, new CompactionConfig().setWait(false).setIterators(Arrays.asList(iterConfig))); try { // this compaction should fail because previous one set iterators c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); if (System.currentTimeMillis() - t1 < 2000) Assert.fail( "Expected compaction to fail because another concurrent compaction set iterators"); } catch (AccumuloException e) {} }
conn.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); conn.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
@Override protected void doTableOp() throws Exception { c.tableOperations().compact(table, new CompactionConfig()); } }));
@Override public void compact(String tableName, CompactionConfig config) throws AccumuloSecurityException, TableNotFoundException, AccumuloException { if (!exists(tableName)) throw new TableNotFoundException(tableName, tableName, ""); if (config.getIterators().size() > 0 || config.getCompactionStrategy() != null) throw new UnsupportedOperationException( "Mock does not support iterators or compaction strategies for compactions"); }
@Override public Void run() throws Exception { Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken()); // Shouldn't throw an exception since we granted the create table permission final String table = testName.getMethodName() + "_user_table"; conn.tableOperations().create(table); // Make sure we can actually use the table we made BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); Mutation m = new Mutation("a"); m.put("b", "c", "d"); bw.addMutation(m); bw.close(); conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true)); return null; } });
@Test public void testDropA() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); writeFlush(c, tableName, "a"); writeFlush(c, tableName, "b"); // create a file that starts with A containing rows 'a' and 'b' c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); writeFlush(c, tableName, "c"); writeFlush(c, tableName, "d"); // drop files that start with A CompactionStrategyConfig csConfig = new CompactionStrategyConfig( TestCompactionStrategy.class.getName()); csConfig.setOptions(ImmutableMap.of("dropPrefix", "A", "inputPrefix", "F")); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig)); Assert.assertEquals(ImmutableSet.of("c", "d"), getRows(c, tableName)); // this compaction should not drop files starting with A c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); Assert.assertEquals(ImmutableSet.of("c", "d"), getRows(c, tableName)); }
bw.close(); conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true)); return null;
@Override protected void doTableOp(final Shell shellState, final String tableName) throws AccumuloException, AccumuloSecurityException { // compact the tables if (cancel) { try { shellState.getConnector().tableOperations().cancelCompaction(tableName); Shell.log.info("Compaction canceled for table " + tableName); } catch (TableNotFoundException e) { throw new AccumuloException(e); } } else { try { if (compactionConfig.getWait()) { Shell.log.info("Compacting table ..."); } for (IteratorSetting iteratorSetting : compactionConfig.getIterators()) { ScanCommand.ensureTserversCanLoadIterator(shellState, tableName, iteratorSetting.getIteratorClass()); } shellState.getConnector().tableOperations().compact(tableName, compactionConfig); Shell.log.info("Compaction of table " + tableName + " " + (compactionConfig.getWait() ? "completed" : "started") + " for given range"); } catch (Exception ex) { throw new AccumuloException(ex); } } }
compactionConfig = new CompactionConfig(); compactionConfig.setFlush(!cl.hasOption(noFlushOption.getOpt())); compactionConfig.setWait(cl.hasOption(waitOpt.getOpt())); compactionConfig.setStartRow(OptUtil.getStartRow(cl)); compactionConfig.setEndRow(OptUtil.getEndRow(cl)); compactionConfig.setIterators(new ArrayList<>(iterators)); compactionConfig.setCompactionStrategy(csc); "org.apache.accumulo.tserver.compaction.strategies.ConfigurableCompactionStrategy"); csc.setOptions(configurableCompactOpt); compactionConfig.setCompactionStrategy(csc);
@Override public Void run() throws Exception { Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken()); // Make sure we can actually use the table we made // Write data final long ts = 1000l; BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); Mutation m = new Mutation("a"); m.put("b", "c", new ColumnVisibility(viz.getBytes()), ts, "d"); bw.addMutation(m); bw.close(); // Compact conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true)); // Alter conn.tableOperations().setProperty(table, Property.TABLE_BLOOM_ENABLED.getKey(), "true"); // Read (and proper authorizations) Scanner s = conn.createScanner(table, new Authorizations(viz)); Iterator<Entry<Key,Value>> iter = s.iterator(); assertTrue("No results from iterator", iter.hasNext()); Entry<Key,Value> entry = iter.next(); assertEquals(new Key("a", "b", "c", viz, ts), entry.getKey()); assertEquals(new Value("d".getBytes()), entry.getValue()); assertFalse("Had more results from iterator", iter.hasNext()); return null; } });
@Test public void testFileSize() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); // write random data because its very unlikely it will compress writeRandomValue(c, tableName, 1 << 16); writeRandomValue(c, tableName, 1 << 16); writeRandomValue(c, tableName, 1 << 9); writeRandomValue(c, tableName, 1 << 7); writeRandomValue(c, tableName, 1 << 6); Assert.assertEquals(5, FunctionalTestUtils.countRFiles(c, tableName)); CompactionStrategyConfig csConfig = new CompactionStrategyConfig( SizeCompactionStrategy.class.getName()); csConfig.setOptions(ImmutableMap.of("size", "" + (1 << 15))); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig)); Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName)); csConfig = new CompactionStrategyConfig(SizeCompactionStrategy.class.getName()); csConfig.setOptions(ImmutableMap.of("size", "" + (1 << 17))); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig)); Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName)); }
for (IteratorSetting setting : config.getIterators()) { String iteratorClass = setting.getIteratorClass(); if (!testClassLoad(tableName, iteratorClass, skviName)) { final String compactionStrategyName = config.getCompactionStrategy().getClassName(); if (!CompactionStrategyConfigUtil.DEFAULT_STRATEGY.getClassName() .equals(compactionStrategyName)) { Text start = config.getStartRow(); Text end = config.getEndRow(); if (config.getFlush()) _flush(tableId, start, end, true); start == null ? EMPTY : TextUtil.getByteBuffer(start), end == null ? EMPTY : TextUtil.getByteBuffer(end), ByteBuffer.wrap(IteratorUtil.encodeIteratorSettings(config.getIterators())), ByteBuffer.wrap(CompactionStrategyConfigUtil.encode(config.getCompactionStrategy()))); doFateOperation(FateOperation.TABLE_COMPACT, args, opts, tableName, config.getWait()); } catch (TableExistsException e) {
conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true));
@Override public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException, AccumuloException { compact(tableName, new CompactionConfig().setStartRow(start).setEndRow(end) .setIterators(iterators).setFlush(flush).setWait(wait)); }