conn.tableOperations().create(table.getFullTableName()); conn.tableOperations().create(table.getIndexTableName()); conn.tableOperations().create(table.getMetricsTableName()); conn.tableOperations().attachIterator(table.getMetricsTableName(), s); Indexer indexer = new Indexer(conn, new Authorizations(), table, new BatchWriterConfig()); indexer.index(m1); indexer.flush(); Scanner scan = conn.createScanner(table.getIndexTableName(), new Authorizations()); scan.setRange(new Range()); Iterator<Entry<Key, Value>> iter = scan.iterator(); assertKeyValuePair(iter.next(), AGE_VALUE, "cf_age", "row1", ""); assertKeyValuePair(iter.next(), bytes("abc"), "cf_arr", "row1", ""); assertFalse(iter.hasNext()); scan.close(); scan = conn.createScanner(table.getMetricsTableName(), new Authorizations());
public boolean exists(String table) { return connector.tableOperations().exists(table); }
public void deleteAccumuloTable(String tableName) { try { connector.tableOperations().delete(tableName); } catch (AccumuloException | AccumuloSecurityException e) { throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to delete Accumulo table", e); } catch (TableNotFoundException e) { throw new PrestoException(ACCUMULO_TABLE_DNE, "Failed to delete Accumulo table, does not exist", e); } }
public void setIterator(String table, IteratorSetting setting) { try { // Remove any existing iterator settings of the same name, if applicable Map<String, EnumSet<IteratorScope>> iterators = connector.tableOperations().listIterators(table); if (iterators.containsKey(setting.getName())) { connector.tableOperations().removeIterator(table, setting.getName(), iterators.get(setting.getName())); } connector.tableOperations().attachIterator(table, setting); } catch (AccumuloSecurityException | AccumuloException e) { throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to set iterator on table " + table, e); } catch (TableNotFoundException e) { throw new PrestoException(ACCUMULO_TABLE_DNE, "Failed to set iterator, table does not exist", e); } }
private void writeData(String tableName, Connector conn) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, MutationsRejectedException { TreeSet<Text> splits = new TreeSet<>(); for (int i = 1; i < 100; i++) { splits.add(new Text(String.format("%06d", i * 100))); } conn.tableOperations().create(tableName); conn.tableOperations().addSplits(tableName, splits); BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig()); for (int i = 0; i < 100; i++) { String row = String.format("%06d", i * 100 + 3); Mutation m = new Mutation(row); m.put("cf1", "cq1", "1"); bw.addMutation(m); } bw.close(); }
String tableId = connector.tableOperations().tableIdMap().get(table); Scanner scanner = connector.createScanner("accumulo.metadata", auths); scanner.fetchColumnFamily(new Text("loc")); Key start = new Key(tableId); Key end = defaultTabletRow.followingKey(PartialKey.ROW); scanner.setRange(new Range(start, end)); Iterator<Entry<Key, Value>> iter = scanner.iterator(); if (iter.hasNext()) { location = Optional.of(iter.next().getValue().toString()); Text splitCompareKey = new Text(); key.getRow(splitCompareKey); Text scannedCompareKey = new Text();
@Override public void run() { try { TreeSet<Text> splits = new TreeSet<>(); splits.add(new Text("X")); conn.tableOperations().addSplits(tableName, splits); } catch (Exception e) { ex.set(e); } } };
boolean isOffline(String tablename, Connector connector) throws TableNotFoundException { String tableId = connector.tableOperations().tableIdMap().get(tablename); Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.setRange(new Range(new Text(tableId + ";"), new Text(tableId + "<"))); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); return Iterators.size(scanner.iterator()) == 0; }
private void reassignLocation(String table, String tableNameToModify) throws TableNotFoundException, MutationsRejectedException { String tableIdToModify = getConnector().tableOperations().tableIdMap().get(tableNameToModify); Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY); scanner.setRange(new KeyExtent(tableIdToModify, null, null).toMetadataRange()); scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME); Entry<Key,Value> entry = scanner.iterator().next(); Mutation m = new Mutation(entry.getKey().getRow()); m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getTimestamp()); m.put(entry.getKey().getColumnFamily(), new Text("1234567"), entry.getKey().getTimestamp() + 1, new Value("fake:9005".getBytes(UTF_8))); scanner.close(); BatchWriter bw = getConnector().createBatchWriter(table, null); bw.addMutation(m); bw.close(); }
static void runTest(Connector c, MiniAccumuloClusterImpl cluster) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, MutationsRejectedException, IOException, InterruptedException, NoSuchAlgorithmException { c.tableOperations().create(tablename); BatchWriter bw = c.createBatchWriter(tablename, new BatchWriterConfig()); for (int i = 0; i < 10; i++) { Mutation m = new Mutation("" + i); m.put(input_cf, input_cq, "row" + i); bw.addMutation(m); } bw.close(); Process hash = cluster.exec(RowHash.class, Collections.singletonList(hadoopTmpDirArg), "-i", c.getInstance().getInstanceName(), "-z", c.getInstance().getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "-t", tablename, "--column", input_cfcq); assertEquals(0, hash.waitFor()); Scanner s = c.createScanner(tablename, Authorizations.EMPTY); s.fetchColumn(new Text(input_cf), new Text(output_cq)); int i = 0; for (Entry<Key,Value> entry : s) { MessageDigest md = MessageDigest.getInstance("MD5"); byte[] check = Base64.encodeBase64(md.digest(("row" + i).getBytes())); assertEquals(entry.getValue().toString(), new String(check)); i++; } } }
private Optional<String> getDefaultTabletLocation(String fulltable) { try { String tableId = connector.tableOperations().tableIdMap().get(fulltable); // Create a scanner over the metadata table, fetching the 'loc' column of the default tablet row Scanner scan = connector.createScanner("accumulo.metadata", connector.securityOperations().getUserAuthorizations(username)); scan.fetchColumnFamily(new Text("loc")); scan.setRange(new Range(tableId + '<')); // scan the entry Optional<String> location = Optional.empty(); for (Entry<Key, Value> entry : scan) { if (location.isPresent()) { throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "Scan for default tablet returned more than one entry"); } location = Optional.of(entry.getValue().toString()); } scan.close(); return location; } catch (Exception e) { // Swallow this exception so the query does not fail due to being unable to locate the tablet server for the default tablet. // This is purely an optimization, but we will want to log the error. LOG.error("Failed to get tablet location, returning dummy location", e); return Optional.empty(); } }
private void addDuplicateLocation(String table, String tableNameToModify) throws TableNotFoundException, MutationsRejectedException { String tableIdToModify = getConnector().tableOperations().tableIdMap().get(tableNameToModify); Mutation m = new Mutation(new KeyExtent(tableIdToModify, null, null).getMetadataEntry()); m.put(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME, new Text("1234567"), new Value("fake:9005".getBytes(UTF_8))); BatchWriter bw = getConnector().createBatchWriter(table, null); bw.addMutation(m); bw.close(); }
public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException { if (simulate) { log.info("Simulating adding table: " + tableName); return; } log.debug("Adding table: " + tableName); BatchWriter bw = null; String table = tableName.toString(); if (createTables && !client.tableOperations().exists(table)) { try { client.tableOperations().create(table); } catch (AccumuloSecurityException e) { log.error("Accumulo security violation creating " + table, e); throw e; } catch (TableExistsException e) { // Shouldn't happen } } try { bw = mtbw.getBatchWriter(table); } catch (TableNotFoundException e) { log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e); throw new AccumuloException(e); } catch (AccumuloException | AccumuloSecurityException e) { throw e; } if (bw != null) bws.put(tableName, bw); }
if (!verbose) { for (Text row : maxSplits > 0 ? shellState.getAccumuloClient().tableOperations().listSplits(tableName, maxSplits) : shellState.getAccumuloClient().tableOperations().listSplits(tableName)) { p.print(encode(encode, row)); Authorizations.EMPTY); TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); final Text start = new Text( shellState.getAccumuloClient().tableOperations().tableIdMap().get(tableName)); final Text end = new Text(start); end.append(new byte[] {'<'}, 0, 1); scanner.setRange(new Range(start, end)); for (Iterator<Entry<Key,Value>> iterator = scanner.iterator(); iterator.hasNext();) { final Entry<Key,Value> next = iterator.next(); if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(next.getKey())) {
private void createTable(String t, boolean online) throws AccumuloSecurityException, AccumuloException, TableNotFoundException, TableExistsException { Connector conn = getConnector(); conn.tableOperations().create(t); conn.tableOperations().online(t, true); SortedSet<Text> partitionKeys = new TreeSet<>(); partitionKeys.add(new Text("some split")); conn.tableOperations().addSplits(t, partitionKeys); if (!online) { conn.tableOperations().offline(t, true); } }
public static synchronized DistributedQueryRunner createAccumuloQueryRunner(Map<String, String> extraProperties) throws Exception { DistributedQueryRunner queryRunner = new DistributedQueryRunner(createSession(), 4, extraProperties); queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); queryRunner.installPlugin(new AccumuloPlugin()); Map<String, String> accumuloProperties = ImmutableMap.<String, String>builder() .put(AccumuloConfig.INSTANCE, connector.getInstance().getInstanceName()) .put(AccumuloConfig.ZOOKEEPERS, connector.getInstance().getZooKeepers()) .put(AccumuloConfig.USERNAME, MAC_USER) .put(AccumuloConfig.PASSWORD, MAC_PASSWORD) .put(AccumuloConfig.ZOOKEEPER_METADATA_ROOT, "/presto-accumulo-test") .build(); queryRunner.createCatalog("accumulo", "accumulo", accumuloProperties); if (!tpchLoaded) { copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), TpchTable.getTables()); connector.tableOperations().addSplits("tpch.orders", ImmutableSortedSet.of(new Text(new LexicoderRowSerializer().encode(BIGINT, 7500L)))); tpchLoaded = true; } return queryRunner; }
@Test public void testPreSplit() throws Exception { String tableName = getUniqueNames(1)[0]; Connector c = getConnector(); c.tableOperations().create(tableName); SortedSet<Text> splits = new TreeSet<>(); splits.add(new Text("8")); splits.add(new Text("256")); c.tableOperations().addSplits(tableName, splits); runTest(c, tableName); }
@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception { final String tableName = OptUtil.getTableOpt(cl, shellState); final boolean decode = cl.hasOption(base64Opt.getOpt()); final TreeSet<Text> splits = new TreeSet<>(); if (cl.hasOption(optSplitsFile.getOpt())) { splits.addAll(ShellUtil.scanFile(cl.getOptionValue(optSplitsFile.getOpt()), decode)); } else { if (cl.getArgList().isEmpty()) { throw new MissingArgumentException("No split points specified"); } for (String s : cl.getArgs()) { splits.add(new Text(s.getBytes(Shell.CHARSET))); } } if (!shellState.getAccumuloClient().tableOperations().exists(tableName)) { throw new TableNotFoundException(null, tableName, null); } shellState.getAccumuloClient().tableOperations().addSplits(tableName, splits); return 0; }
Map<Table.ID,String> tableIdToName = invert(tableNameToId); for (String table : tops.list()) { if (MetadataTable.NAME.equals(table) || RootTable.NAME.equals(table)) { continue; propertiesForTable = tops.getProperties(table); } catch (TableNotFoundException e) { log.warn("Could not fetch properties for {}", table, e); WorkSection.limit(bs); try { Text buffer = new Text(); for (Entry<Key,Value> entry : bs) { Key k = entry.getKey();
@Test(timeout = 2 * 60 * 1000) public void test() throws Exception { final Connector conn = this.getConnector(); // Yes, there's a tabletserver assertEquals(1, conn.instanceOperations().getTabletServers().size()); final String tableName = getUniqueNames(1)[0]; conn.tableOperations().create(tableName); BatchWriter bw = conn.createBatchWriter(tableName, null); for (int i = 0; i < N; i++) { Mutation m = new Mutation("" + i); m.put("", "", ""); bw.addMutation(m); } bw.close(); conn.tableOperations().flush(tableName, null, null, true); // Kill dfs cluster.getMiniDfs().restartNameNode(false); assertEquals(N, Iterators.size(conn.createScanner(tableName, Authorizations.EMPTY).iterator())); }