Refine search
scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(new Text(startkey), null)); scanner.addScanIterator(cfg); scanner.fetchColumn(colFam, new Text(field)); final Text cq = new Text(); for (Entry<Key, Value> rowEntry : row.entrySet()) { rowEntry.getKey().getColumnQualifier(cq); rowData.put(cq.toString(), new ByteArrayByteIterator(rowEntry.getValue().get())); } finally { if (null != scanner) { scanner.close();
String tableId = connector.tableOperations().tableIdMap().get(table); Scanner scanner = connector.createScanner("accumulo.metadata", auths); scanner.fetchColumnFamily(new Text("loc")); Key defaultTabletRow = new Key(tableId + '<'); Key start = new Key(tableId); Key end = defaultTabletRow.followingKey(PartialKey.ROW); scanner.setRange(new Range(start, end)); Iterator<Entry<Key, Value>> iter = scanner.iterator(); if (iter.hasNext()) { location = Optional.of(iter.next().getValue().toString()); Text splitCompareKey = new Text(); key.getRow(splitCompareKey); Text scannedCompareKey = new Text(); location = Optional.of(entry.getValue().toString()); break; int compareTo = splitCompareKey.compareTo(scannedCompareKey); if (compareTo <= 0) { location = Optional.of(entry.getValue().toString()); scanner.close();
/** * Gets a scanner from Accumulo over one row. * * @param row the row to scan * @param fields the set of columns to scan * @return an Accumulo {@link Scanner} bound to the given row and columns */ private Scanner getRow(String table, Text row, Set<String> fields) throws TableNotFoundException { Scanner scanner = connector.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(row)); if (fields != null) { for (String field : fields) { scanner.fetchColumn(colFam, new Text(field)); } } return scanner; }
public static Pair<byte[], byte[]> getMinMaxRowIds(Connector connector, AccumuloTable table, Authorizations auths) throws TableNotFoundException { Scanner scanner = connector.createScanner(table.getMetricsTableName(), auths); scanner.setRange(new Range(new Text(Indexer.METRICS_TABLE_ROW_ID.array()))); Text family = new Text(Indexer.METRICS_TABLE_ROWS_CF.array()); Text firstRowQualifier = new Text(Indexer.METRICS_TABLE_FIRST_ROW_CQ.array()); Text lastRowQualifier = new Text(Indexer.METRICS_TABLE_LAST_ROW_CQ.array()); scanner.fetchColumn(family, firstRowQualifier); scanner.fetchColumn(family, lastRowQualifier); byte[] firstRow = null; byte[] lastRow = null; for (Entry<Key, Value> entry : scanner) { if (entry.getKey().compareColumnQualifier(firstRowQualifier) == 0) { firstRow = entry.getValue().get(); } if (entry.getKey().compareColumnQualifier(lastRowQualifier) == 0) { lastRow = entry.getValue().get(); } } scanner.close(); return Pair.of(firstRow, lastRow); }
private long getNumRowsInTable(String metricsTable, Authorizations auths) throws TableNotFoundException { // Create scanner against the metrics table, pulling the special column and the rows column Scanner scanner = connector.createScanner(metricsTable, auths); scanner.setRange(METRICS_TABLE_ROWID_RANGE); scanner.fetchColumn(METRICS_TABLE_ROWS_CF_AS_TEXT, CARDINALITY_CQ_AS_TEXT); // Scan the entry and get the number of rows long numRows = -1; for (Entry<Key, Value> entry : scanner) { if (numRows > 0) { throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "Should have received only one entry when scanning for number of rows in metrics table"); } numRows = Long.parseLong(entry.getValue().toString()); } scanner.close(); LOG.debug("Number of rows in table is %d", numRows); return numRows; }
@Override public Status read(String table, String key, Set<String> fields, Map<String, ByteIterator> result) { Scanner scanner = null; try { scanner = getRow(table, new Text(key), null); // Pick out the results we care about. final Text cq = new Text(); for (Entry<Key, Value> entry : scanner) { entry.getKey().getColumnQualifier(cq); Value v = entry.getValue(); byte[] buf = v.get(); result.put(cq.toString(), new ByteArrayByteIterator(buf)); } } catch (Exception e) { System.err.println("Error trying to reading Accumulo table " + table + " " + key); e.printStackTrace(); return Status.ERROR; } finally { if (null != scanner) { scanner.close(); } } return Status.OK; }
@Test public void run() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); for (int i = 0; i < 100000; i++) { Mutation m = new Mutation(new Text(String.format("%08d", i))); for (int j = 0; j < 3; j++) m.put(new Text("cf1"), new Text("cq" + j), new Value((i + "_" + j).getBytes(UTF_8))); bw.addMutation(m); } bw.close(); Scanner scanner = c.createScanner(tableName, new Authorizations()); scanner.setBatchSize(1000); Iterator<Entry<Key,Value>> iter = scanner.iterator(); verify(iter, 0, 200); // sleep three times the session timeout sleepUninterruptibly(9, TimeUnit.SECONDS); verify(iter, 200, 100000); }
public void deleteProblemReports(Table.ID table) throws Exception { if (isMeta(table)) { Iterator<ProblemReport> pri = iterator(table); while (pri.hasNext()) { pri.next().removeFromZooKeeper(context); } return; } Scanner scanner = context.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.addScanIterator(new IteratorSetting(1, "keys-only", SortedKeyIterator.class)); scanner.setRange(new Range(new Text("~err_" + table))); Mutation delMut = new Mutation(new Text("~err_" + table)); boolean hasProblems = false; for (Entry<Key,Value> entry : scanner) { hasProblems = true; delMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier()); } if (hasProblems) MetadataTableUtil.getMetadataTable(context).update(delMut); }
private void reassignLocation(String table, String tableNameToModify) throws TableNotFoundException, MutationsRejectedException { String tableIdToModify = getConnector().tableOperations().tableIdMap().get(tableNameToModify); Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY); scanner.setRange(new KeyExtent(tableIdToModify, null, null).toMetadataRange()); scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME); Entry<Key,Value> entry = scanner.iterator().next(); Mutation m = new Mutation(entry.getKey().getRow()); m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getTimestamp()); m.put(entry.getKey().getColumnFamily(), new Text("1234567"), entry.getKey().getTimestamp() + 1, new Value("fake:9005".getBytes(UTF_8))); scanner.close(); BatchWriter bw = getConnector().createBatchWriter(table, null); bw.addMutation(m); bw.close(); }
private Optional<String> getDefaultTabletLocation(String fulltable) { try { String tableId = connector.tableOperations().tableIdMap().get(fulltable); // Create a scanner over the metadata table, fetching the 'loc' column of the default tablet row Scanner scan = connector.createScanner("accumulo.metadata", connector.securityOperations().getUserAuthorizations(username)); scan.fetchColumnFamily(new Text("loc")); scan.setRange(new Range(tableId + '<')); // scan the entry Optional<String> location = Optional.empty(); for (Entry<Key, Value> entry : scan) { if (location.isPresent()) { throw new PrestoException(FUNCTION_IMPLEMENTATION_ERROR, "Scan for default tablet returned more than one entry"); } location = Optional.of(entry.getValue().toString()); } scan.close(); return location; } catch (Exception e) { // Swallow this exception so the query does not fail due to being unable to locate the tablet server for the default tablet. // This is purely an optimization, but we will want to log the error. LOG.error("Failed to get tablet location, returning dummy location", e); return Optional.empty(); } }
protected int removeReplicationEntries(Map<UUID,TServerInstance> candidates) { try { try { final Scanner s = ReplicationTable.getScanner(context); StatusSection.limit(s); for (Entry<Key,Value> entry : s) { UUID id = path2uuid(new Path(entry.getKey().getRow().toString())); candidates.remove(id); log.info("Ignore closed log " + id + " because it is being replicated"); } } catch (ReplicationTableOfflineException ex) { return candidates.size(); } final Scanner scanner = context.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.fetchColumnFamily(MetadataSchema.ReplicationSection.COLF); scanner.setRange(MetadataSchema.ReplicationSection.getRange()); for (Entry<Key,Value> entry : scanner) { Text file = new Text(); MetadataSchema.ReplicationSection.getFile(entry.getKey(), file); UUID id = path2uuid(new Path(file.toString())); candidates.remove(id); log.info("Ignore closed log " + id + " because it is being replicated"); } return candidates.size(); } catch (TableNotFoundException e) { log.error("Failed to scan metadata table", e); throw new IllegalArgumentException(e); } }
@Override public String getNamespace(final String pfx) throws RyaDAOException { try { final Scanner scanner = connector.createScanner(tableLayoutStrategy.getNs(), ALL_AUTHORIZATIONS); scanner.fetchColumn(INFO_NAMESPACE_TXT, EMPTY_TEXT); scanner.setRange(new Range(new Text(pfx))); final Iterator<Map.Entry<Key, Value>> iterator = scanner .iterator(); if (iterator.hasNext()) { return new String(iterator.next().getValue().get(), StandardCharsets.UTF_8); } } catch (final Exception e) { throw new RyaDAOException(e); } return null; }
@Test public void unclosedWalsLeaveStatusOpen() throws Exception { Set<String> wals = Collections.emptySet(); BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); Mutation m = new Mutation( ReplicationSection.getRowPrefix() + "file:/accumulo/wal/tserver+port/12345"); m.put(ReplicationSection.COLF, new Text("1"), StatusUtil.fileCreatedValue(System.currentTimeMillis())); bw.addMutation(m); bw.close(); refs.updateReplicationEntries(conn, wals); Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY); s.fetchColumnFamily(ReplicationSection.COLF); Entry<Key,Value> entry = Iterables.getOnlyElement(s); Status status = Status.parseFrom(entry.getValue().get()); Assert.assertFalse(status.getClosed()); }
private static Text findInitialEnd(Scanner scanner) { Text end = new Text(new byte[] {(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff}); scanner.setRange(new Range(end, null)); while (scanner.iterator().hasNext()) { Text t = new Text(); t.append(end.getBytes(), 0, end.getLength()); t.append(end.getBytes(), 0, end.getLength()); end = t; scanner.setRange(new Range(end, null)); } return end; }
private void runMergeTest(Connector conn, String table, String[] splits, String[] inserts, String start, String end, String last, long expected) throws Exception { log.info("table " + table); conn.tableOperations().create(table, new NewTableConfiguration().setTimeType(TimeType.LOGICAL)); TreeSet<Text> splitSet = new TreeSet<>(); for (String split : splits) { splitSet.add(new Text(split)); } conn.tableOperations().addSplits(table, splitSet); BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig()); for (String row : inserts) { Mutation m = new Mutation(row); m.put("cf", "cq", "v"); bw.addMutation(m); } bw.flush(); conn.tableOperations().merge(table, start == null ? null : new Text(start), end == null ? null : new Text(end)); Mutation m = new Mutation(last); m.put("cf", "cq", "v"); bw.addMutation(m); bw.flush(); Scanner scanner = conn.createScanner(table, Authorizations.EMPTY); scanner.setRange(new Range(last)); bw.close(); long time = scanner.iterator().next().getKey().getTimestamp(); if (time != expected) throw new RuntimeException("unexpected time " + time + " " + expected); }
ms.setRange(new KeyExtent(tableId, null, null).toMetadataRange()); ms.fetchColumnFamily(DataFileColumnFamily.NAME); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(ms); Key key = cell.getKey(); if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) { FileRef ref = new FileRef(context.getVolumeManager(), key); bw.addMutation(createDeleteMutation(context, tableId, ref.meta().toString())); bw.addMutation(createDeleteMutation(context, tableId, cell.getValue().toString())); bw.flush(); ms.clearColumns(); m = new Mutation(key.getRow()); if (lock != null) putLockID(context, lock, m); if (key.getRow().compareTo(m.getRow(), 0, m.getRow().length) != 0) { bw.addMutation(m); m = new Mutation(key.getRow());
public static SortedMap<FileRef,DataFileValue> getDataFileSizes(KeyExtent extent, ServerContext context) { TreeMap<FileRef,DataFileValue> sizes = new TreeMap<>(); try (Scanner mdScanner = new ScannerImpl(context, MetadataTable.ID, Authorizations.EMPTY)) { mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME); Text row = extent.getMetadataEntry(); Key endKey = new Key(row, DataFileColumnFamily.NAME, new Text("")); endKey = endKey.followingKey(PartialKey.ROW_COLFAM); mdScanner.setRange(new Range(new Key(row), endKey)); for (Entry<Key,Value> entry : mdScanner) { if (!entry.getKey().getRow().equals(row)) break; DataFileValue dfv = new DataFileValue(entry.getValue().get()); sizes.put(new FileRef(context.getVolumeManager(), entry.getKey()), dfv); } return sizes; } }
public static void removeBulkLoadEntries(Connector conn, String tableId, long tid) throws Exception { Scanner mscanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS)); mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange()); mscanner.fetchColumnFamily(Constants.METADATA_BULKFILE_COLUMN_FAMILY); BatchWriter bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig()); for (Entry<Key,Value> entry : mscanner) { log.debug("Looking at entry " + entry + " with tid " + tid); if (Long.parseLong(entry.getValue().toString()) == tid) { log.debug("deleting entry " + entry); Mutation m = new Mutation(entry.getKey().getRow()); m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier()); bw.addMutation(m); } } bw.close(); }
@Override protected void resetScanner() { try { Scanner ds = conn.createScanner(metadataTableName, Authorizations.EMPTY); Text tablet = new KeyExtent("0", new Text("m"), null).getMetadataEntry(); ds.setRange(new Range(tablet, true, tablet, true)); Mutation m = new Mutation(tablet); BatchWriter bw = conn.createBatchWriter(metadataTableName, new BatchWriterConfig()); for (Entry<Key,Value> entry : ds) { Key k = entry.getKey(); m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp()); } bw.addMutation(m); bw.close(); } catch (Exception e) { throw new RuntimeException(e); } super.resetScanner(); }
scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); scanner.setRange(TabletsSection.getRange(getServicedTableId())); Text colf = new Text(); Text colq = new Text(); if (!row.equals(entry.getKey().getRow())) { currentKeyExtent = null; location = null; row = entry.getKey().getRow(); row = entry.getKey().getRow(); location = null; } else if (colf.equals(TabletsSection.CurrentLocationColumnFamily.NAME)) { location = entry.getValue().toString();