public static synchronized DistributedQueryRunner createAccumuloQueryRunner(Map<String, String> extraProperties) throws Exception { DistributedQueryRunner queryRunner = new DistributedQueryRunner(createSession(), 4, extraProperties); queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); queryRunner.installPlugin(new AccumuloPlugin()); Map<String, String> accumuloProperties = ImmutableMap.<String, String>builder() .put(AccumuloConfig.INSTANCE, connector.getInstance().getInstanceName()) .put(AccumuloConfig.ZOOKEEPERS, connector.getInstance().getZooKeepers()) .put(AccumuloConfig.USERNAME, MAC_USER) .put(AccumuloConfig.PASSWORD, MAC_PASSWORD) .put(AccumuloConfig.ZOOKEEPER_METADATA_ROOT, "/presto-accumulo-test") .build(); queryRunner.createCatalog("accumulo", "accumulo", accumuloProperties); if (!tpchLoaded) { copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), TpchTable.getTables()); connector.tableOperations().addSplits("tpch.orders", ImmutableSortedSet.of(new Text(new LexicoderRowSerializer().encode(BIGINT, 7500L)))); tpchLoaded = true; } return queryRunner; }
@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception { final String tableName = OptUtil.getTableOpt(cl, shellState); final boolean decode = cl.hasOption(base64Opt.getOpt()); final TreeSet<Text> splits = new TreeSet<>(); if (cl.hasOption(optSplitsFile.getOpt())) { splits.addAll(ShellUtil.scanFile(cl.getOptionValue(optSplitsFile.getOpt()), decode)); } else { if (cl.getArgList().isEmpty()) { throw new MissingArgumentException("No split points specified"); } for (String s : cl.getArgs()) { splits.add(new Text(s.getBytes(Shell.CHARSET))); } } if (!shellState.getAccumuloClient().tableOperations().exists(tableName)) { throw new TableNotFoundException(null, tableName, null); } shellState.getAccumuloClient().tableOperations().addSplits(tableName, splits); return 0; }
private void addSplits(TableOperations opts, String... points) throws Exception { SortedSet<Text> splits = new TreeSet<>(); for (String point : points) { splits.add(new Text(point)); } opts.addSplits(MetadataTable.NAME, splits); }
@Override public void run() { try { TreeSet<Text> splits = new TreeSet<>(); splits.add(new Text("X")); conn.tableOperations().addSplits(tableName, splits); } catch (Exception e) { ex.set(e); } } };
public void addSplits(Connector connector, String tableName) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { // Add 10 splits to the table SortedSet<Text> partitions = new TreeSet<>(); for (String s : "b,e,g,j,l,o,q,t,v,y".split(",")) partitions.add(new Text(s)); connector.tableOperations().addSplits(tableName, partitions); }
private void splitTable(TableOperations tOps, String tableName) throws Exception { SortedSet<Text> splits = new TreeSet<>(); for (int i = 1; i <= 9; i++) { splits.add(new Text(itos(i * (ROWS / 10)))); } log.debug("Adding splits to " + tableName); tOps.addSplits(tableName, splits); } }
/** Copy the splits placed on table t1 to table t2. */ public static void copySplits(TableOperations tops, String t1, String t2) throws AccumuloException { try { Collection<Text> splits = tops.listSplits(t1); SortedSet<Text> ss = new TreeSet<>(splits); tops.addSplits(t2, ss); } catch (TableNotFoundException | AccumuloSecurityException e) { log.error("cannot handle splits copying from "+t1+" to "+t2, e); throw new RuntimeException(e); } }
public void addSplit(String tableName, SortedSet<Text> partitions) { try { connector.tableOperations().addSplits(tableName, partitions); } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) { log.warn("",e); } } public Instance getInstance() {
@Override public void run() { try { // split the table final SortedSet<Text> afterEnd = SPLITS.tailSet(new Text(end.toString() + "\0")); conn.tableOperations().addSplits(tableName, afterEnd); } catch (Exception ex) { log.error("Exception", ex); synchronized (fail) { fail[0] = true; } } } };
@Override public void run() { try { getConnector().tableOperations().addSplits(finalName, splits); } catch (TableNotFoundException ex) { // expected, ignore } catch (Exception ex) { throw new RuntimeException(finalName, ex); } } });
@Override protected void runLater(State state, Environment env) throws Exception { SortedSet<Text> splits = new TreeSet<>(); Random rand = (Random) state.get("rand"); int count = rand.nextInt(20); for (int i = 0; i < count; i++) splits.add(new Text(String.format(BulkPlusOne.FMT, (rand.nextLong() & 0x7fffffffffffffffl) % BulkPlusOne.LOTS))); log.info("splitting " + splits); env.getConnector().tableOperations().addSplits(Setup.getTableName(), splits); log.info("split for " + splits + " finished"); }
@Test(expected = AccumuloException.class) public void testRootTableSplit() throws Exception { TableOperations opts = getConnector().tableOperations(); SortedSet<Text> splits = new TreeSet<>(); splits.add(new Text("5")); opts.addSplits(RootTable.NAME, splits); }
@Override public void visit(State state, Environment env, Properties props) throws Exception { String table = state.getString("tableName"); Random rand = (Random) state.get("rand"); Connector conn = env.getConnector(); String row = Utils.getBank(rand.nextInt((Integer) state.get("numBanks"))); log.debug("adding split " + row); conn.tableOperations().addSplits(table, new TreeSet<>(Arrays.asList(new Text(row)))); }
@Test public void testPreSplit() throws Exception { String tableName = getUniqueNames(1)[0]; Connector c = getConnector(); c.tableOperations().create(tableName); SortedSet<Text> splits = new TreeSet<>(); splits.add(new Text("8")); splits.add(new Text("256")); c.tableOperations().addSplits(tableName, splits); runTest(c, tableName); }
@Override public void addSplits(ByteBuffer login, String tableName, Set<ByteBuffer> splits) throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, org.apache.accumulo.proxy.thrift.TableNotFoundException, TException { try { SortedSet<Text> sorted = new TreeSet<>(); for (ByteBuffer split : splits) { sorted.add(ByteBufferUtil.toText(split)); } getConnector(login).tableOperations().addSplits(tableName, sorted); } catch (Exception e) { handleExceptionTNF(e); } }
@After public void restoreMetadataSplits() throws Exception { if (null != metadataSplits) { log.info("Restoring split on metadata table"); Connector conn = getConnector(); conn.tableOperations().merge(MetadataTable.NAME, null, null); conn.tableOperations().addSplits(MetadataTable.NAME, new TreeSet<>(metadataSplits)); } }
@Override public void visit(State state, Environment env, Properties props) throws Exception { String indexTableName = (String) state.get("indexTableName"); int numPartitions = (Integer) state.get("numPartitions"); Random rand = (Random) state.get("rand"); SortedSet<Text> splitSet = ShardFixture.genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x"); log.debug("adding splits " + indexTableName); env.getConnector().tableOperations().addSplits(indexTableName, splitSet); }
private void createTable(String t, boolean online) throws AccumuloSecurityException, AccumuloException, TableNotFoundException, TableExistsException { Connector conn = getConnector(); conn.tableOperations().create(t); conn.tableOperations().online(t, true); SortedSet<Text> partitionKeys = new TreeSet<>(); partitionKeys.add(new Text("some split")); conn.tableOperations().addSplits(t, partitionKeys); if (!online) { conn.tableOperations().offline(t, true); } }
private void createSeqTable(String table) { // create tables if they don't exist if (!connector.tableOperations().exists(table)) { IteratorSetting longCombiner = new IteratorSetting(1, SummingCombiner.class); SummingCombiner.setCombineAllColumns(longCombiner, true); SummingCombiner.setEncodingType(longCombiner, LongLexicoderTemp.class); try { connector.tableOperations().create(table); connector.tableOperations().addSplits(table, getSplitPoints(3)); GraphuloUtil.applyIteratorSoft(longCombiner, connector.tableOperations(), table); } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException | TableExistsException e) { log.warn("", e); } } }
private MiniAccumuloCluster createMiniAccumuloWithTestTableAndSplits(SortedSet<Text> sortedSet) throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException { MiniAccumuloCluster accumuloCluster; accumuloCluster = new MiniAccumuloCluster(Files.createTempDir(), PASSWORD); accumuloCluster.start(); Connector connector = accumuloCluster.getConnector(USERNAME, PASSWORD); TableOperations tableOperations = connector.tableOperations(); tableOperations.create(TABLE_NAME); tableOperations.addSplits(TABLE_NAME, sortedSet); return accumuloCluster; }