public static void configureScanner(ScannerBase scanner, CurrentState state) { TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); scanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME); scanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME); scanner .fetchColumnFamily(TabletsSection.SuspendLocationColumn.SUSPEND_COLUMN.getColumnFamily()); scanner.fetchColumnFamily(LogColumnFamily.NAME); scanner.fetchColumnFamily(ChoppedColumnFamily.NAME); scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class)); IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class); if (state != null) { TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers()); TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables()); TabletStateChangeIterator.setMerges(tabletChange, state.merges()); TabletStateChangeIterator.setMigrations(tabletChange, state.migrationsSnapshot()); TabletStateChangeIterator.setMasterState(tabletChange, state.getMasterState()); TabletStateChangeIterator.setShuttingDown(tabletChange, state.shutdownServers()); } scanner.addScanIterator(tabletChange); }
@Override public void close() { if (scannerBase != null) { scannerBase.close(); } }
protected void fetchColumns(final CommandLine cl, final ScannerBase scanner, final ScanInterpreter formatter) throws UnsupportedEncodingException { if (cl.hasOption(scanOptColumns.getOpt())) { for (String a : cl.getOptionValue(scanOptColumns.getOpt()).split(",")) { final String sa[] = a.split(":", 2); if (sa.length == 1) { scanner.fetchColumnFamily( formatter.interpretColumnFamily(new Text(a.getBytes(Shell.CHARSET)))); } else { scanner.fetchColumn( formatter.interpretColumnFamily(new Text(sa[0].getBytes(Shell.CHARSET))), formatter.interpretColumnQualifier(new Text(sa[1].getBytes(Shell.CHARSET)))); } } } }
if (c.getSecond() != null) { log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond()); scannerBase.fetchColumn(c.getFirst(), c.getSecond()); } else { log.debug("Fetching column family " + c.getFirst()); scannerBase.fetchColumnFamily(c.getFirst()); scannerBase.setSamplerConfiguration(samplerConfig); scannerIterator = scannerBase.iterator(); numKeysRead = 0;
/** * Sets the option on this currently running resource. * * @param options * @return */ @Override public AccumuloResource setOptions(SessionOptions options) { // set the families for (Column family : options.getFetchedColumns()) { if (family.columnQualifier != null) baseScanner.fetchColumn(new Text(family.columnFamily), new Text(family.columnQualifier)); else { if (log.isTraceEnabled()) log.trace("Setting column family " + new Text(family.columnFamily)); baseScanner.fetchColumnFamily(new Text(family.columnFamily)); } } for (IteratorSetting setting : options.getIterators()) { if (log.isTraceEnabled()) log.trace("Adding setting, " + setting); baseScanner.addScanIterator(setting); } return this; }
/** * Configures the iterators on a scanner for the given table name. * * @param job * the Hadoop job configuration * @param scanner * the scanner for which to configure the iterators * @param tableName * the table name for which the scanner is configured * @since 1.7.0 */ private void setupIterators(JobConf job, ScannerBase scanner, String tableName, org.apache.accumulo.core.client.mapreduce.RangeInputSplit split) { List<IteratorSetting> iterators = null; if (split == null) { iterators = jobIterators(job, tableName); } else { iterators = split.getIterators(); if (iterators == null) { iterators = jobIterators(job, tableName); } } for (IteratorSetting iterator : iterators) scanner.addScanIterator(iterator); }
@Override protected Iterator<Map.Entry<Key, Value>> retrieveIterator() { return scanner.iterator(); } }
); EdgeRefFilter.setVertexIds(edgeRefFilterSettings, vertexIdsSet); scanner.addScanIterator(edgeRefFilterSettings); VertexEdgeIdIterator.class ); scanner.addScanIterator(vertexEdgeIdIteratorSettings); Iterator<Map.Entry<Key, Value>> it = scanner.iterator(); List<String> edgeIds = new ArrayList<>(); while (it.hasNext()) { scanner.close(); GRAPH_LOGGER.logEndIterator(System.currentTimeMillis() - timerStartTime);
/** * Limit the scanner to only return Status records */ public static void limit(ScannerBase scanner) { scanner.fetchColumnFamily(NAME); }
private void applyFetchHints(ScannerBase scanner, FetchHints fetchHints, ElementType elementType) { scanner.clearColumns(); Iterable<Text> columnFamiliesToFetch = getColumnFamiliesToFetch(elementType, fetchHints); for (Text columnFamilyToFetch : columnFamiliesToFetch) { scanner.fetchColumnFamily(columnFamilyToFetch); } }
private void testScanner(ScannerBase s, long expected) { long now = System.currentTimeMillis(); try { s.iterator().next(); } finally { s.close(); } long diff = System.currentTimeMillis() - now; log.info("Diff = {}", diff); assertTrue("Scanner taking too long to return intermediate results: " + diff, diff < expected); } }
private void assertSampleNotPresent(SamplerConfiguration sc, ScannerBase... scanners) { for (ScannerBase scanner : scanners) { SamplerConfiguration csc = scanner.getSamplerConfiguration(); scanner.setSamplerConfiguration(sc); try { for (Iterator<Entry<Key,Value>> i = scanner.iterator(); i.hasNext();) { Entry<Key,Value> entry = i.next(); entry.getKey(); } Assert.fail("Expected SampleNotPresentException, but it did not happen : " + scanner.getClass().getSimpleName()); } catch (SampleNotPresentException e) { } scanner.clearSamplerConfiguration(); for (Iterator<Entry<Key,Value>> i = scanner.iterator(); i.hasNext();) { Entry<Key,Value> entry = i.next(); entry.getKey(); } if (csc == null) { scanner.clearSamplerConfiguration(); } else { scanner.setSamplerConfiguration(csc); } } }
public void fetch(ScannerBase sb) { sb.fetchColumn(colf, colq); }
if (pos1 == -1) { // no ranges - collection of singleton texts for (Text text : GraphuloUtil.d4mRowToTexts(colFilter)) { scanner.fetchColumn(GraphuloUtil.EMPTY_TEXT, text); scanner.addScanIterator(s);
protected void setupSampling(final String tableName, final CommandLine cl, final Shell shellState, ScannerBase scanner) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { if (getUseSample(cl)) { SamplerConfiguration samplerConfig = shellState.getAccumuloClient().tableOperations() .getSamplerConfiguration(tableName); if (samplerConfig == null) { throw new SampleNotPresentException( "Table " + tableName + " does not have sampling configured"); } Shell.log.debug("Using sampling configuration : " + samplerConfig); scanner.setSamplerConfiguration(samplerConfig); } }
sb.clearSamplerConfiguration(); Assert.assertEquals(20000, Iterables.size(sb)); sb.setSamplerConfiguration(SC1);
@Override public void clearColumns() { delegate.clearColumns(); }
if (c.getSecond() != null) { log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond()); scannerBase.fetchColumn(c.getFirst(), c.getSecond()); } else { log.debug("Fetching column family " + c.getFirst()); scannerBase.fetchColumnFamily(c.getFirst()); scannerBase.setSamplerConfiguration(samplerConfig); scannerIterator = scannerBase.iterator(); numKeysRead = 0;
protected void fillScanner(ScannerBase scanner, RyaURI context, String qualifier, Long ttl, Long currentTime, TripleRowRegex tripleRowRegex, RdfCloudTripleStoreConfiguration conf) throws IOException { if (context != null && qualifier != null) { scanner.fetchColumn(new Text(context.getData()), new Text(qualifier)); } else if (context != null) { scanner.fetchColumnFamily(new Text(context.getData())); } else if (qualifier != null) { IteratorSetting setting = new IteratorSetting(8, "riq", RegExFilter.class.getName()); RegExFilter.setRegexs(setting, null, null, qualifier, null, false); scanner.addScanIterator(setting); TimestampFilter.setEnd(setting, currentTime, true); scanner.addScanIterator(setting); String regex = tripleRowRegex.getRow(); RegExFilter.setRegexs(setting, regex, null, null, null, false); scanner.addScanIterator(setting); scanner.addScanIterator(itr);
/** * Configures the iterators on a scanner for the given table name. Will attempt to use * configuration from the InputSplit, on failure will try to extract them from * TaskAttemptContext. * * @param context * the Hadoop context for the configured job * @param tableName * the table name for which the scanner is configured * @param scanner * the scanner for which to configure the iterators * @param split * InputSplit containing configurations * @since 1.7.0 */ private void setupIterators(TaskAttemptContext context, ScannerBase scanner, String tableName, RangeInputSplit split) { List<IteratorSetting> iterators = null; if (split == null) { iterators = contextIterators(context, tableName); } else { iterators = split.getIterators(); if (iterators == null) { iterators = contextIterators(context, tableName); } } for (IteratorSetting iterator : iterators) scanner.addScanIterator(iterator); }