compactionConfig = new CompactionConfig();
@Override public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException, AccumuloException { compact(tableName, new CompactionConfig().setStartRow(start).setEndRow(end) .setIterators(iterators).setFlush(flush).setWait(wait)); }
@Override protected void doTableOp() throws Exception { c.tableOperations().compact(table, new CompactionConfig()); } }));
compactionConfig = new CompactionConfig();
@Override public void compact(String tableName, Text start, Text end, List<IteratorSetting> iterators, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException, AccumuloException { compact(tableName, new CompactionConfig().setStartRow(start).setEndRow(end) .setIterators(iterators).setFlush(flush).setWait(wait)); }
bw.close(); conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true)); return null;
@Test public void testDropA() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); writeFlush(c, tableName, "a"); writeFlush(c, tableName, "b"); // create a file that starts with A containing rows 'a' and 'b' c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); writeFlush(c, tableName, "c"); writeFlush(c, tableName, "d"); // drop files that start with A CompactionStrategyConfig csConfig = new CompactionStrategyConfig( TestCompactionStrategy.class.getName()); csConfig.setOptions(ImmutableMap.of("dropPrefix", "A", "inputPrefix", "F")); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig)); Assert.assertEquals(ImmutableSet.of("c", "d"), getRows(c, tableName)); // this compaction should not drop files starting with A c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); Assert.assertEquals(ImmutableSet.of("c", "d"), getRows(c, tableName)); }
@Test public void testConcurrent() throws Exception { // two compactions without iterators or strategy should be able to run concurrently Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); // write random data because its very unlikely it will compress writeRandomValue(c, tableName, 1 << 16); writeRandomValue(c, tableName, 1 << 16); c.tableOperations().compact(tableName, new CompactionConfig().setWait(false)); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName)); writeRandomValue(c, tableName, 1 << 16); IteratorSetting iterConfig = new IteratorSetting(30, SlowIterator.class); SlowIterator.setSleepTime(iterConfig, 1000); long t1 = System.currentTimeMillis(); c.tableOperations().compact(tableName, new CompactionConfig().setWait(false).setIterators(Arrays.asList(iterConfig))); try { // this compaction should fail because previous one set iterators c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); if (System.currentTimeMillis() - t1 < 2000) Assert.fail( "Expected compaction to fail because another concurrent compaction set iterators"); } catch (AccumuloException e) {} }
private void testDropNone(Map<String,String> options) throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); writeFlush(c, tableName, "a"); writeFlush(c, tableName, "b"); CompactionStrategyConfig csConfig = new CompactionStrategyConfig( TestCompactionStrategy.class.getName()); csConfig.setOptions(options); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig)); Assert.assertEquals(ImmutableSet.of("a", "b"), getRows(c, tableName)); }
conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true));
@Override public Void run() throws Exception { Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken()); // Make sure we can actually use the table we made // Write data final long ts = 1000l; BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); Mutation m = new Mutation("a"); m.put("b", "c", new ColumnVisibility(viz.getBytes()), ts, "d"); bw.addMutation(m); bw.close(); // Compact conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true)); // Alter conn.tableOperations().setProperty(table, Property.TABLE_BLOOM_ENABLED.getKey(), "true"); // Read (and proper authorizations) Scanner s = conn.createScanner(table, new Authorizations(viz)); Iterator<Entry<Key,Value>> iter = s.iterator(); assertTrue("No results from iterator", iter.hasNext()); Entry<Key,Value> entry = iter.next(); assertEquals(new Key("a", "b", "c", viz, ts), entry.getKey()); assertEquals(new Value("d".getBytes()), entry.getValue()); assertFalse("Had more results from iterator", iter.hasNext()); return null; } });
@Test public void testPerTableClasspath() throws Exception { // Can't assume that a test-resource will be on the server's classpath Assume.assumeTrue(ClusterType.MINI == getClusterType()); // test per-table classpath + user specified compaction strategy final Connector c = getConnector(); final String tableName = getUniqueNames(1)[0]; File target = new File(System.getProperty("user.dir"), "target"); Assert.assertTrue(target.mkdirs() || target.isDirectory()); File destFile = installJar(target, "/TestCompactionStrat.jar"); c.tableOperations().create(tableName); c.instanceOperations().setProperty( Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "context1", destFile.toString()); c.tableOperations().setProperty(tableName, Property.TABLE_CLASSPATH.getKey(), "context1"); c.tableOperations().addSplits(tableName, new TreeSet<>(Arrays.asList(new Text("efg")))); writeFlush(c, tableName, "a"); writeFlush(c, tableName, "b"); writeFlush(c, tableName, "h"); writeFlush(c, tableName, "i"); Assert.assertEquals(4, FunctionalTestUtils.countRFiles(c, tableName)); // EfgCompactionStrat will only compact a tablet w/ end row of 'efg'. No other tablets are // compacted. CompactionStrategyConfig csConfig = new CompactionStrategyConfig( "org.apache.accumulo.test.EfgCompactionStrat"); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig)); Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName)); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); Assert.assertEquals(2, FunctionalTestUtils.countRFiles(c, tableName)); }
@Test public void testFileSize() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); // write random data because its very unlikely it will compress writeRandomValue(c, tableName, 1 << 16); writeRandomValue(c, tableName, 1 << 16); writeRandomValue(c, tableName, 1 << 9); writeRandomValue(c, tableName, 1 << 7); writeRandomValue(c, tableName, 1 << 6); Assert.assertEquals(5, FunctionalTestUtils.countRFiles(c, tableName)); CompactionStrategyConfig csConfig = new CompactionStrategyConfig( SizeCompactionStrategy.class.getName()); csConfig.setOptions(ImmutableMap.of("size", "" + (1 << 15))); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig)); Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName)); csConfig = new CompactionStrategyConfig(SizeCompactionStrategy.class.getName()); csConfig.setOptions(ImmutableMap.of("size", "" + (1 << 17))); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig)); Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName)); }
@Test public void testIterators() throws Exception { // test compaction strategy + iterators Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); writeFlush(c, tableName, "a"); writeFlush(c, tableName, "b"); // create a file that starts with A containing rows 'a' and 'b' c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); writeFlush(c, tableName, "c"); writeFlush(c, tableName, "d"); Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName)); // drop files that start with A CompactionStrategyConfig csConfig = new CompactionStrategyConfig( TestCompactionStrategy.class.getName()); csConfig.setOptions(ImmutableMap.of("inputPrefix", "F")); IteratorSetting iterConf = new IteratorSetting(21, "myregex", RegExFilter.class); RegExFilter.setRegexs(iterConf, "a|c", null, null, null, false); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true) .setCompactionStrategy(csConfig).setIterators(Arrays.asList(iterConf))); // compaction strategy should only be applied to one file. If its applied to both, then row 'b' // would be dropped by filter. Assert.assertEquals(ImmutableSet.of("a", "b", "c"), getRows(c, tableName)); Assert.assertEquals(2, FunctionalTestUtils.countRFiles(c, tableName)); c.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); // ensure that iterator is not applied Assert.assertEquals(ImmutableSet.of("a", "b", "c"), getRows(c, tableName)); Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName)); }
/** * Creates a table, adds a record to it, and then compacts the table. A simple way to make sure * that the system user exists (since the master does an RPC to the tserver which will create the * system user if it doesn't already exist). */ private void createReadWriteDrop(Connector conn) throws TableNotFoundException, AccumuloSecurityException, AccumuloException, TableExistsException { final String table = testName.getMethodName() + "_table"; conn.tableOperations().create(table); BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); Mutation m = new Mutation("a"); m.put("b", "c", "d"); bw.addMutation(m); bw.close(); conn.tableOperations().compact(table, new CompactionConfig().setFlush(true).setWait(true)); Scanner s = conn.createScanner(table, Authorizations.EMPTY); Entry<Key,Value> entry = Iterables.getOnlyElement(s); assertEquals("Did not find the expected key", 0, new Key("a", "b", "c").compareTo(entry.getKey(), PartialKey.ROW_COLFAM_COLQUAL)); assertEquals("d", entry.getValue().toString()); conn.tableOperations().delete(table); } }
/** * Creates a table, adds a record to it, and then compacts the table. A simple way to make sure * that the system user exists (since the master does an RPC to the tserver which will create the * system user if it doesn't already exist). */ private void createTableWithDataAndCompact(Connector conn) throws TableNotFoundException, AccumuloSecurityException, AccumuloException, TableExistsException { final String table = testName.getMethodName() + "_table"; conn.tableOperations().create(table); BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); Mutation m = new Mutation("a"); m.put("b", "c", "d"); bw.addMutation(m); bw.close(); conn.tableOperations().compact(table, new CompactionConfig().setFlush(true).setWait(true)); } }
@Override public Void run() throws Exception { Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken()); // Shouldn't throw an exception since we granted the create table permission final String table = testName.getMethodName() + "_user_table"; conn.tableOperations().create(table); // Make sure we can actually use the table we made BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); Mutation m = new Mutation("a"); m.put("b", "c", "d"); bw.addMutation(m); bw.close(); conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true)); return null; } });
@Override public void compactTable(ByteBuffer login, String tableName, ByteBuffer startRow, ByteBuffer endRow, List<org.apache.accumulo.proxy.thrift.IteratorSetting> iterators, boolean flush, boolean wait, CompactionStrategyConfig compactionStrategy) throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, org.apache.accumulo.proxy.thrift.TableNotFoundException, org.apache.accumulo.proxy.thrift.AccumuloException, TException { try { CompactionConfig compactionConfig = new CompactionConfig() .setStartRow(ByteBufferUtil.toText(startRow)).setEndRow(ByteBufferUtil.toText(endRow)) .setIterators(getIteratorSettings(iterators)).setFlush(flush).setWait(wait); if (compactionStrategy != null) { // @formatter:off org.apache.accumulo.core.client.admin.CompactionStrategyConfig ccc = new org.apache.accumulo.core.client.admin.CompactionStrategyConfig( compactionStrategy.getClassName()); // @formatter:on if (compactionStrategy.options != null) ccc.setOptions(compactionStrategy.options); compactionConfig.setCompactionStrategy(ccc); } getConnector(login).tableOperations().compact(tableName, compactionConfig); } catch (Exception e) { handleExceptionTNF(e); } }
conn.tableOperations().compact(tableName, new CompactionConfig().setWait(true)); conn.tableOperations().compact(tableName, new CompactionConfig().setWait(true));