@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException, ConstraintViolationException { shellState.checkTableState(); final Mutation m = new Mutation(new Text(cl.getArgs()[0].getBytes(Shell.CHARSET))); final Text colf = new Text(cl.getArgs()[1].getBytes(Shell.CHARSET)); final Text colq = new Text(cl.getArgs()[2].getBytes(Shell.CHARSET)); if (cl.hasOption(deleteOptAuths.getOpt())) { final ColumnVisibility le = new ColumnVisibility(cl.getOptionValue(deleteOptAuths.getOpt())); if (cl.hasOption(timestampOpt.getOpt())) { m.putDelete(colf, colq, le, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt()))); } else { m.putDelete(colf, colq, le); } } else if (cl.hasOption(timestampOpt.getOpt())) { m.putDelete(colf, colq, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt()))); } else { m.putDelete(colf, colq); } final BatchWriter bw = shellState.getAccumuloClient().createBatchWriter( shellState.getTableName(), new BatchWriterConfig().setMaxMemory(Math.max(m.estimatedMemoryUsed(), 1024)) .setMaxWriteThreads(1).setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS)); bw.addMutation(m); bw.close(); return 0; }
public BatchWriterConfig getBatchWriterConfig() { BatchWriterConfig config = new BatchWriterConfig(); config.setMaxWriteThreads(this.batchThreads); config.setMaxLatency(this.batchLatency, TimeUnit.MILLISECONDS); config.setMaxMemory(this.batchMemory); config.setTimeout(this.batchTimeout, TimeUnit.MILLISECONDS); return config; }
.setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS); if (cl.hasOption(durabilityOption.getOpt())) { String userDurability = cl.getOptionValue(durabilityOption.getOpt());
public BatchWriterConfig getBatchWriterConfig() { ensureOpen(); if (batchWriterConfig == null) { Properties props = info.getProperties(); batchWriterConfig = new BatchWriterConfig(); Long maxMemory = ClientProperty.BATCH_WRITER_MEMORY_MAX.getBytes(props); if (maxMemory != null) { batchWriterConfig.setMaxMemory(maxMemory); } Long maxLatency = ClientProperty.BATCH_WRITER_LATENCY_MAX.getTimeInMillis(props); if (maxLatency != null) { batchWriterConfig.setMaxLatency(maxLatency, TimeUnit.SECONDS); } Long timeout = ClientProperty.BATCH_WRITER_TIMEOUT_MAX.getTimeInMillis(props); if (timeout != null) { batchWriterConfig.setTimeout(timeout, TimeUnit.SECONDS); } String durability = ClientProperty.BATCH_WRITER_DURABILITY.getValue(props); if (!durability.isEmpty()) { batchWriterConfig.setDurability(Durability.valueOf(durability.toUpperCase())); } } return batchWriterConfig; }
@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception { final String tableName = OptUtil.getTableOpt(cl, shellState); final ScanInterpreter interpeter = getInterpreter(cl, tableName, shellState); // handle first argument, if present, the authorizations list to // scan with final Authorizations auths = getAuths(cl, shellState); final Scanner scanner = shellState.getAccumuloClient().createScanner(tableName, auths); scanner.addScanIterator( new IteratorSetting(Integer.MAX_VALUE, "NOVALUE", SortedKeyIterator.class)); // handle session-specific scan iterators addScanIterators(shellState, cl, scanner, tableName); // handle remaining optional arguments scanner.setRange(getRange(cl, interpeter)); scanner.setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS); // handle columns fetchColumns(cl, scanner, interpeter); // output / delete the records final BatchWriter writer = shellState.getAccumuloClient().createBatchWriter(tableName, new BatchWriterConfig().setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS)); FormatterConfig config = new FormatterConfig(); config.setPrintTimestamps(cl.hasOption(timestampOpt.getOpt())); shellState.printLines( new DeleterFormatter(writer, scanner, config, shellState, cl.hasOption(forceOpt.getOpt())), false); return 0; }
@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException, ConstraintViolationException { shellState.checkTableState(); final Mutation m = new Mutation(new Text(cl.getArgs()[0].getBytes(Shell.CHARSET))); final Text colf = new Text(cl.getArgs()[1].getBytes(Shell.CHARSET)); final Text colq = new Text(cl.getArgs()[2].getBytes(Shell.CHARSET)); if (cl.hasOption(deleteOptAuths.getOpt())) { final ColumnVisibility le = new ColumnVisibility(cl.getOptionValue(deleteOptAuths.getOpt())); if (cl.hasOption(timestampOpt.getOpt())) { m.putDelete(colf, colq, le, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt()))); } else { m.putDelete(colf, colq, le); } } else if (cl.hasOption(timestampOpt.getOpt())) { m.putDelete(colf, colq, Long.parseLong(cl.getOptionValue(timestampOpt.getOpt()))); } else { m.putDelete(colf, colq); } final BatchWriter bw = shellState.getConnector().createBatchWriter(shellState.getTableName(), new BatchWriterConfig().setMaxMemory(Math.max(m.estimatedMemoryUsed(), 1024)) .setMaxWriteThreads(1).setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS)); bw.addMutation(m); bw.close(); return 0; }
public BatchWriterConfig getBatchWriterConfig() { BatchWriterConfig config = new BatchWriterConfig(); config.setMaxWriteThreads(this.batchThreads); config.setMaxLatency(this.batchLatency, TimeUnit.MILLISECONDS); config.setMaxMemory(this.batchMemory); config.setTimeout(this.batchTimeout, TimeUnit.MILLISECONDS); return config; }
private void initBatchWriter() { ClientConfiguration cc = ClientConfiguration.loadDefault().withInstance(instanceName) .withZkHosts(zookeeperHost).withZkTimeout(zookeeperTimeout); Instance instance = new ZooKeeperInstance(cc); try { connector = instance.getConnector(username, auth); } catch (Exception e) { log.error("failed to connect to Accumulo instance " + instanceName, e); throw new RuntimeException(e); } BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxMemory(batchWriterMaxMemory); bwc.setTimeout(batchWriterTimeout, TimeUnit.SECONDS); try { batchWriter = connector.createBatchWriter(tableName, bwc); } catch (TableNotFoundException e) { log.error(tableName + " does not exist in instance " + instanceName, e); throw new RuntimeException(e); } }
batchWriterConfig.setTimeout(RdfCloudTripleStoreConstants.MAX_TIME, TimeUnit.MILLISECONDS); batchWriterConfig.setMaxWriteThreads(RdfCloudTripleStoreConstants.NUM_THREADS); writer = conn.createMultiTableBatchWriter(batchWriterConfig);
public BatchWriterConfig createBatchWriterConfig() { long maxMemory = getConfigLong(BATCHWRITER_MAX_MEMORY, DEFAULT_BATCHWRITER_MAX_MEMORY); long maxLatency = getConfigLong(BATCHWRITER_MAX_LATENCY, DEFAULT_BATCHWRITER_MAX_LATENCY); int maxWriteThreads = getInt(BATCHWRITER_MAX_WRITE_THREADS, DEFAULT_BATCHWRITER_MAX_WRITE_THREADS); long timeout = getConfigLong(BATCHWRITER_TIMEOUT, DEFAULT_BATCHWRITER_TIMEOUT); BatchWriterConfig config = new BatchWriterConfig(); config.setMaxMemory(maxMemory); config.setMaxLatency(maxLatency, TimeUnit.MILLISECONDS); config.setMaxWriteThreads(maxWriteThreads); config.setTimeout(timeout, TimeUnit.MILLISECONDS); return config; }
/** * Create a {@link BatchWriterConfig} based on this configuration. * @return */ public BatchWriterConfig getBatchWriterConfig() { return new BatchWriterConfig().setMaxLatency(getMaxWriteLatency(), TimeUnit.MILLISECONDS) .setMaxMemory(getMaxWriteMemory()) .setMaxWriteThreads(getMaxWriteThreads()) .setTimeout(getMaxWriteTimeout(), TimeUnit.MILLISECONDS); }
/** * Create a {@link BatchWriterConfig} based on this configuration. * @return */ public BatchWriterConfig getBatchWriterConfig() { return new BatchWriterConfig().setMaxLatency(getMaxWriteLatency(), TimeUnit.MILLISECONDS) .setMaxMemory(getMaxWriteMemory()) .setMaxWriteThreads(getMaxWriteThreads()) .setTimeout(getMaxWriteTimeout(), TimeUnit.MILLISECONDS); }
public BatchWriterConfig createBatchWriterConfig() { long maxMemory = getConfigLong(BATCHWRITER_MAX_MEMORY, DEFAULT_BATCHWRITER_MAX_MEMORY); long maxLatency = getConfigLong(BATCHWRITER_MAX_LATENCY, DEFAULT_BATCHWRITER_MAX_LATENCY); int maxWriteThreads = getInt(BATCHWRITER_MAX_WRITE_THREADS, DEFAULT_BATCHWRITER_MAX_WRITE_THREADS); long timeout = getConfigLong(BATCHWRITER_TIMEOUT, DEFAULT_BATCHWRITER_TIMEOUT); BatchWriterConfig config = new BatchWriterConfig(); config.setMaxMemory(maxMemory); config.setMaxLatency(maxLatency, TimeUnit.MILLISECONDS); config.setMaxWriteThreads(maxWriteThreads); config.setTimeout(timeout, TimeUnit.MILLISECONDS); return config; } }
public BatchWriterConfig createBatchWriterConfig() { long maxMemory = getConfigLong(BATCHWRITER_MAX_MEMORY, DEFAULT_BATCHWRITER_MAX_MEMORY); long maxLatency = getConfigLong(BATCHWRITER_MAX_LATENCY, DEFAULT_BATCHWRITER_MAX_LATENCY); int maxWriteThreads = getInt(BATCHWRITER_MAX_WRITE_THREADS, DEFAULT_BATCHWRITER_MAX_WRITE_THREADS); long timeout = getConfigLong(BATCHWRITER_TIMEOUT, DEFAULT_BATCHWRITER_TIMEOUT); BatchWriterConfig config = new BatchWriterConfig(); config.setMaxMemory(maxMemory); config.setMaxLatency(maxLatency, TimeUnit.MILLISECONDS); config.setMaxWriteThreads(maxWriteThreads); config.setTimeout(timeout, TimeUnit.MILLISECONDS); return config; }
@Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception { final String tableName = OptUtil.getTableOpt(cl, shellState); final ScanInterpreter interpeter = getInterpreter(cl, tableName, shellState); // handle first argument, if present, the authorizations list to // scan with final Authorizations auths = getAuths(cl, shellState); final Scanner scanner = shellState.getConnector().createScanner(tableName, auths); scanner.addScanIterator( new IteratorSetting(Integer.MAX_VALUE, "NOVALUE", SortedKeyIterator.class)); // handle session-specific scan iterators addScanIterators(shellState, cl, scanner, tableName); // handle remaining optional arguments scanner.setRange(getRange(cl, interpeter)); scanner.setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS); // handle columns fetchColumns(cl, scanner, interpeter); // output / delete the records final BatchWriter writer = shellState.getConnector().createBatchWriter(tableName, new BatchWriterConfig().setTimeout(getTimeout(cl), TimeUnit.MILLISECONDS)); FormatterConfig config = new FormatterConfig(); config.setPrintTimestamps(cl.hasOption(timestampOpt.getOpt())); shellState.printLines( new DeleterFormatter(writer, scanner, config, shellState, cl.hasOption(forceOpt.getOpt())), false); return 0; }
/** * Gets the {@link BatchWriterConfig} settings that were stored with ClientInfo */ public static BatchWriterConfig getBatchWriterOptions(Class<?> implementingClass, Configuration conf) { BatchWriterConfig bwConfig = new BatchWriterConfig(); Properties props = getClientProperties(implementingClass, conf); String property = props.getProperty(BATCH_WRITER_DURABILITY.getKey()); if (property != null) bwConfig.setDurability(DurabilityImpl.fromString(property)); Long value = BATCH_WRITER_LATENCY_MAX.getTimeInMillis(props); if (value != null) bwConfig.setMaxLatency(value, TimeUnit.MILLISECONDS); value = BATCH_WRITER_MEMORY_MAX.getBytes(props); if (value != null) bwConfig.setMaxMemory(value); value = BATCH_WRITER_TIMEOUT_MAX.getTimeInMillis(props); if (value != null) bwConfig.setTimeout(value, TimeUnit.MILLISECONDS); Integer intValue = BATCH_WRITER_THREADS_MAX.getInteger(props); if (intValue != null) bwConfig.setMaxWriteThreads(intValue); return bwConfig; }
public void testBatchWriterTimeout(Connector conn, String tableName) throws Exception { conn.tableOperations().create(tableName); conn.tableOperations().addConstraint(tableName, SlowConstraint.class.getName()); // give constraint time to propagate through zookeeper sleepUninterruptibly(1, TimeUnit.SECONDS); BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig().setTimeout(3, TimeUnit.SECONDS)); Mutation mut = new Mutation("r1"); mut.put("cf1", "cq1", "v1"); bw.addMutation(mut); try { bw.close(); fail("batch writer did not timeout"); } catch (MutationsRejectedException mre) { if (mre.getCause() instanceof TimedOutException) return; throw mre; } }
BatchWriterPlusProblem getWriter(ByteBuffer login, String tableName, WriterOptions opts) throws Exception { BatchWriterConfig cfg = new BatchWriterConfig(); if (opts != null) { if (opts.maxMemory != 0) cfg.setMaxMemory(opts.maxMemory); if (opts.threads != 0) cfg.setMaxWriteThreads(opts.threads); if (opts.timeoutMs != 0) cfg.setTimeout(opts.timeoutMs, TimeUnit.MILLISECONDS); if (opts.latencyMs != 0) cfg.setMaxLatency(opts.latencyMs, TimeUnit.MILLISECONDS); if (opts.isSetDurability() && opts.durability != null) { cfg.setDurability(getDurability(opts.getDurability())); } } BatchWriterPlusProblem result = new BatchWriterPlusProblem(); result.writer = getConnector(login).createBatchWriter(tableName, cfg); return result; }
batchWriterConfig = new BatchWriterConfig(); batchWriterConfig.setMaxMemory(MAX_MEMORY); batchWriterConfig.setTimeout(MAX_TIME, TimeUnit.MILLISECONDS); batchWriterConfig.setMaxWriteThreads(NUM_THREADS);
batchWriterConfig = new BatchWriterConfig(); batchWriterConfig.setMaxMemory(MAX_MEMORY); batchWriterConfig.setTimeout(MAX_TIME, TimeUnit.MILLISECONDS); batchWriterConfig.setMaxWriteThreads(NUM_THREADS);