@Override public String toString() { StringBuilder sb = new StringBuilder(32); sb.append("[maxMemory=").append(getMaxMemory()).append(", maxLatency=") .append(getMaxLatency(TimeUnit.MILLISECONDS)).append(", maxWriteThreads=") .append(getMaxWriteThreads()).append(", timeout=").append(getTimeout(TimeUnit.MILLISECONDS)) .append(", durability=").append(durability).append("]"); return sb.toString(); } }
public TabletServerBatchWriter(ClientContext context, BatchWriterConfig config) { this.context = context; this.maxMem = config.getMaxMemory(); this.maxLatency = config.getMaxLatency(TimeUnit.MILLISECONDS) <= 0 ? Long.MAX_VALUE : config.getMaxLatency(TimeUnit.MILLISECONDS); this.timeout = config.getTimeout(TimeUnit.MILLISECONDS); this.mutations = new MutationSet(); this.lastProcessingStartTime = System.currentTimeMillis(); this.durability = config.getDurability(); this.writer = new MutationWriter(config.getMaxWriteThreads()); if (this.maxLatency != Long.MAX_VALUE) { jtimer.schedule(new TimerTask() { @Override public void run() { try { synchronized (TabletServerBatchWriter.this) { if ((System.currentTimeMillis() - lastProcessingStartTime) > TabletServerBatchWriter.this.maxLatency) startProcessing(); } } catch (Throwable t) { updateUnknownErrors("Max latency task failed " + t.getMessage(), t); } } }, 0, this.maxLatency / 4); } }
@Override public ConnectionOptions<T> batchWriterConfig(BatchWriterConfig batchWriterConfig) { ClientProperty.BATCH_WRITER_MEMORY_MAX.setBytes(properties, batchWriterConfig.getMaxMemory()); ClientProperty.BATCH_WRITER_LATENCY_MAX.setTimeInMillis(properties, batchWriterConfig.getMaxLatency(TimeUnit.MILLISECONDS)); ClientProperty.BATCH_WRITER_TIMEOUT_MAX.setTimeInMillis(properties, batchWriterConfig.getTimeout(TimeUnit.MILLISECONDS)); setProperty(ClientProperty.BATCH_WRITER_THREADS_MAX, batchWriterConfig.getMaxWriteThreads()); setProperty(ClientProperty.BATCH_WRITER_DURABILITY, batchWriterConfig.getDurability().toString()); return this; }
/** * Create a new MutationBuffer with the given configuration parameters. * * @param config * the configuration for this buffer */ MutationBuffer(ProxyConnector connector, BatchWriterConfig config) { this.connector = connector; this.maxMemory = config.getMaxMemory(); this.maxLatencyMs = config.getMaxLatency(TimeUnit.MILLISECONDS); this.memory = 0L; this.mutations = new HashMap<String,List<Mutation>>(); }
@Override public String toString() { StringBuilder sb = new StringBuilder(32); sb.append("[maxMemory=").append(getMaxMemory()).append(", maxLatency=") .append(getMaxLatency(TimeUnit.MILLISECONDS)).append(", maxWriteThreads=") .append(getMaxWriteThreads()).append(", timeout=").append(getTimeout(TimeUnit.MILLISECONDS)) .append(", durability=").append(durability).append("]"); return sb.toString(); } }
@Override public BatchWriter createBatchWriter(String tableName, BatchWriterConfig config) throws TableNotFoundException { return createBatchWriter(tableName, config.getMaxMemory(), config.getMaxLatency(TimeUnit.MILLISECONDS), config.getMaxWriteThreads()); }
@Override public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, BatchWriterConfig config) throws TableNotFoundException { return createBatchDeleter(tableName, authorizations, numQueryThreads, config.getMaxMemory(), config.getMaxLatency(TimeUnit.MILLISECONDS), config.getMaxWriteThreads()); }
@Override public MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config) { return createMultiTableBatchWriter(config.getMaxMemory(), config.getMaxLatency(TimeUnit.MILLISECONDS), config.getMaxWriteThreads()); }
public TabletServerBatchWriter(ClientContext context, BatchWriterConfig config) { this.context = context; this.maxMem = config.getMaxMemory(); this.maxLatency = config.getMaxLatency(TimeUnit.MILLISECONDS) <= 0 ? Long.MAX_VALUE : config.getMaxLatency(TimeUnit.MILLISECONDS); this.timeout = config.getTimeout(TimeUnit.MILLISECONDS); this.mutations = new MutationSet(); this.lastProcessingStartTime = System.currentTimeMillis(); this.durability = config.getDurability(); this.writer = new MutationWriter(config.getMaxWriteThreads()); if (this.maxLatency != Long.MAX_VALUE) { jtimer.schedule(new TimerTask() { @Override public void run() { try { synchronized (TabletServerBatchWriter.this) { if ((System.currentTimeMillis() - lastProcessingStartTime) > TabletServerBatchWriter.this.maxLatency) startProcessing(); } } catch (Throwable t) { updateUnknownErrors("Max latency task failed " + t.getMessage(), t); } } }, 0, this.maxLatency / 4); } }
@Test public void testBatchWriterConfigUsesDefaultWithNoParameters() { Map configMap = Maps.newHashMap(); AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(configMap); BatchWriterConfig batchWriterConfig = accumuloGraphConfiguration.createBatchWriterConfig(); assertThat(batchWriterConfig.getMaxLatency(TimeUnit.MILLISECONDS), is(AccumuloGraphConfiguration.DEFAULT_BATCHWRITER_MAX_LATENCY)); assertThat(batchWriterConfig.getTimeout(TimeUnit.MILLISECONDS), is(AccumuloGraphConfiguration.DEFAULT_BATCHWRITER_TIMEOUT)); assertThat(batchWriterConfig.getMaxMemory(), is(AccumuloGraphConfiguration.DEFAULT_BATCHWRITER_MAX_MEMORY)); assertThat(batchWriterConfig.getMaxWriteThreads(), is(AccumuloGraphConfiguration.DEFAULT_BATCHWRITER_MAX_WRITE_THREADS)); }
@Test public void testBatchWriterConfigIsSetToValuesWithParameters() { int numThreads = 2; long timeout = 3; long maxMemory = 5; long maxLatency = 7; Map configMap = Maps.newHashMap(); MapUtils.putAll(configMap, new String[]{ AccumuloGraphConfiguration.BATCHWRITER_MAX_LATENCY, "" + maxLatency, AccumuloGraphConfiguration.BATCHWRITER_MAX_MEMORY, "" + maxMemory, AccumuloGraphConfiguration.BATCHWRITER_MAX_WRITE_THREADS, "" + numThreads, AccumuloGraphConfiguration.BATCHWRITER_TIMEOUT, "" + timeout}); AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(configMap); BatchWriterConfig batchWriterConfig = accumuloGraphConfiguration.createBatchWriterConfig(); assertThat(batchWriterConfig.getMaxLatency(TimeUnit.MILLISECONDS), is(maxLatency)); assertThat(batchWriterConfig.getTimeout(TimeUnit.MILLISECONDS), is(timeout)); assertThat(batchWriterConfig.getMaxMemory(), is(maxMemory)); assertThat(batchWriterConfig.getMaxWriteThreads(), is(numThreads)); } }
ProxyBatchWriter(ProxyConnector connector, ByteBuffer token, String table, BatchWriterConfig config) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TException { this.connector = connector; this.client = connector.getClient(); this.token = token; WriterOptions opts = new WriterOptions(); opts.setLatencyMs(config.getMaxLatency(TimeUnit.MILLISECONDS)); opts.setMaxMemory(config.getMaxMemory()); opts.setThreads(config.getMaxWriteThreads()); writerId = client.createWriter(token, table, opts); mutationBuffer = new MutationBuffer(connector, config); closed = false; }