/** * Creates a BatchWriter with the expected configuration. * * @param table The table to write to */ private BatchWriter createBatchWriter(String table) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency( Long.parseLong(getProperties() .getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong( getProperties().getProperty("accumulo.batchWriterSize", "100000"))); final String numThreadsValue = getProperties().getProperty("accumulo.batchWriterThreads"); // Try to saturate the client machine. int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); if (null != numThreadsValue) { numThreads = Integer.parseInt(numThreadsValue); } System.err.println("Using " + numThreads + " threads to write data"); bwc.setMaxWriteThreads(numThreads); return connector.createBatchWriter(table, bwc); }
/** * Creates a BatchWriter with the expected configuration. * * @param table The table to write to */ private BatchWriter createBatchWriter(String table) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency( Long.parseLong(getProperties() .getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong( getProperties().getProperty("accumulo.batchWriterSize", "100000"))); final String numThreadsValue = getProperties().getProperty("accumulo.batchWriterThreads"); // Try to saturate the client machine. int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); if (null != numThreadsValue) { numThreads = Integer.parseInt(numThreadsValue); } System.err.println("Using " + numThreads + " threads to write data"); bwc.setMaxWriteThreads(numThreads); return connector.createBatchWriter(table, bwc); }
/** * Creates a BatchWriter with the expected configuration. * * @param table The table to write to */ private BatchWriter createBatchWriter(String table) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig(); bwc.setMaxLatency( Long.parseLong(getProperties() .getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS); bwc.setMaxMemory(Long.parseLong( getProperties().getProperty("accumulo.batchWriterSize", "100000"))); final String numThreadsValue = getProperties().getProperty("accumulo.batchWriterThreads"); // Try to saturate the client machine. int numThreads = Math.max(1, Runtime.getRuntime().availableProcessors() / 2); if (null != numThreadsValue) { numThreads = Integer.parseInt(numThreadsValue); } System.err.println("Using " + numThreads + " threads to write data"); bwc.setMaxWriteThreads(numThreads); return connector.createBatchWriter(table, bwc); }
private void resetWriter() { BatchWriter writer = null; try { writer = accumuloClient.createBatchWriter(tableName, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS)); } catch (Exception ex) { log.warn("Unable to create a batch writer, will retry. Set log level to" + " DEBUG to see stacktrace. cause: " + ex); log.debug("batch writer creation failed with exception.", ex); } finally { /* Trade in the new writer (even if null) for the one we need to close. */ writer = this.writer.getAndSet(writer); try { if (writer != null) { writer.close(); } } catch (Exception ex) { log.warn( "Problem closing batch writer. Set log level to DEBUG to see stacktrace. cause: " + ex); log.debug("batch writer close failed with exception", ex); } } }
BatchWriter createBatchWriter() { try { return context.createBatchWriter(targetTableName, new BatchWriterConfig().setMaxMemory(MAX_MEMORY) .setMaxLatency(LATENCY, TimeUnit.MILLISECONDS).setMaxWriteThreads(THREADS)); } catch (Exception e) { throw new RuntimeException(e); } }
@Override public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { checkArgument(tableName != null, "tableName is null"); checkArgument(authorizations != null, "authorizations is null"); return new TabletServerBatchDeleter(context, context.getTableId(tableName), authorizations, numQueryThreads, new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
public BatchWriterConfig getBatchWriterConfig() { BatchWriterConfig config = new BatchWriterConfig(); config.setMaxWriteThreads(this.batchThreads); config.setMaxLatency(this.batchLatency, TimeUnit.MILLISECONDS); config.setMaxMemory(this.batchMemory); config.setTimeout(this.batchTimeout, TimeUnit.MILLISECONDS); return config; }
@Override public BatchWriter createBatchWriter(String tableName, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { checkArgument(tableName != null, "tableName is null"); return new BatchWriterImpl(context, context.getTableId(tableName), new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
@Override public MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads) { return new MultiTableBatchWriterImpl(context, new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
conf.get(Property.TRACE_ZK_PATH)); writer = new AtomicReference<>(this.accumuloClient.createBatchWriter(tableName, new BatchWriterConfig().setMaxLatency(BATCH_WRITER_MAX_LATENCY, TimeUnit.SECONDS)));
public BatchWriterConfig getBatchWriterConfig() { ensureOpen(); if (batchWriterConfig == null) { Properties props = info.getProperties(); batchWriterConfig = new BatchWriterConfig(); Long maxMemory = ClientProperty.BATCH_WRITER_MEMORY_MAX.getBytes(props); if (maxMemory != null) { batchWriterConfig.setMaxMemory(maxMemory); } Long maxLatency = ClientProperty.BATCH_WRITER_LATENCY_MAX.getTimeInMillis(props); if (maxLatency != null) { batchWriterConfig.setMaxLatency(maxLatency, TimeUnit.SECONDS); } Long timeout = ClientProperty.BATCH_WRITER_TIMEOUT_MAX.getTimeInMillis(props); if (timeout != null) { batchWriterConfig.setTimeout(timeout, TimeUnit.SECONDS); } String durability = ClientProperty.BATCH_WRITER_DURABILITY.getValue(props); if (!durability.isEmpty()) { batchWriterConfig.setDurability(Durability.valueOf(durability.toUpperCase())); } } return batchWriterConfig; }
BatchWriter bw = new BatchWriterImpl(context, MetadataTable.ID, new BatchWriterConfig().setMaxMemory(1000000) .setMaxLatency(120000L, TimeUnit.MILLISECONDS).setMaxWriteThreads(2))) {
BatchWriter createBatchWriter() { try { return context.getConnector().createBatchWriter(targetTableName, new BatchWriterConfig().setMaxMemory(MAX_MEMORY) .setMaxLatency(LATENCY, TimeUnit.MILLISECONDS).setMaxWriteThreads(THREADS)); } catch (TableNotFoundException e) { // ya, I don't think so throw new RuntimeException(e); } catch (Exception e) { throw new RuntimeException(e); } }
@Override @Deprecated public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { return createBatchDeleter(tableName, authorizations, numQueryThreads, new BatchWriterConfig().setMaxLatency(maxLatency, TimeUnit.MILLISECONDS) .setMaxMemory(maxMemory).setMaxWriteThreads(maxWriteThreads)); }
@Deprecated @Override public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { checkArgument(tableName != null, "tableName is null"); checkArgument(authorizations != null, "authorizations is null"); return new TabletServerBatchDeleter(context, getTableId(tableName), authorizations, numQueryThreads, new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
@Deprecated @Override public BatchWriter createBatchWriter(String tableName, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { checkArgument(tableName != null, "tableName is null"); return new BatchWriterImpl(context, getTableId(tableName), new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
@Deprecated @Override public MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads) { return new MultiTableBatchWriterImpl(context, new BatchWriterConfig().setMaxMemory(maxMemory) .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); }
public BatchWriterConfig getBatchWriterConfig() { BatchWriterConfig config = new BatchWriterConfig(); config.setMaxWriteThreads(this.batchThreads); config.setMaxLatency(this.batchLatency, TimeUnit.MILLISECONDS); config.setMaxMemory(this.batchMemory); config.setTimeout(this.batchTimeout, TimeUnit.MILLISECONDS); return config; }
public BatchWriter createBatchWriter (String table, long maxMemory, long maxLatency,int maxWriteThreads) throws TableNotFoundException { BatchWriterConfig bwc = new BatchWriterConfig() .setMaxLatency(maxLatency, TimeUnit.MILLISECONDS) .setMaxMemory(maxMemory) .setMaxWriteThreads(maxWriteThreads); return connector.createBatchWriter(table, bwc); } public BatchWriter createBatchWriter (String table) throws TableNotFoundException {
/** * Create a {@link BatchWriterConfig} based on this configuration. * @return */ public BatchWriterConfig getBatchWriterConfig() { return new BatchWriterConfig().setMaxLatency(getMaxWriteLatency(), TimeUnit.MILLISECONDS) .setMaxMemory(getMaxWriteMemory()) .setMaxWriteThreads(getMaxWriteThreads()) .setTimeout(getMaxWriteTimeout(), TimeUnit.MILLISECONDS); }