void sealIfFlushNeeded() { long timeSinceFirstEvent = System.currentTimeMillis() - firstEventTimestamp; if (firstEventTimestamp > 0 && timeSinceFirstEvent > emitter.config.getFlushMillis()) { seal(); } }
private void unlockAndSealIfNeeded() { if (eventCount.incrementAndGet() >= emitter.config.getFlushCount()) { unlockAndSeal(); } else { long timeSinceFirstEvent = System.currentTimeMillis() - firstEventTimestamp; if (firstEventTimestamp > 0 && timeSinceFirstEvent > emitter.config.getFlushMillis()) { unlockAndSeal(); } else { unlock(); } } }
@Override public void run() { while (true) { boolean needsToShutdown = needsToShutdown(); try { emitLargeEvents(); emitBatches(); tryEmitOneFailedBuffer(); if (needsToShutdown) { tryEmitAndDrainAllFailedBuffers(); // Make GC life easier drainBuffersToReuse(); return; } } catch (Throwable t) { log.error(t, "Uncaught exception in EmittingThread.run()"); } if (failedBuffers.isEmpty()) { // Waiting for 1/2 of config.getFlushMillis() in order to flush events not more than 50% later than specified. // If nanos=0 parkNanos() doesn't wait at all, then we don't want. long waitNanos = Math.max(TimeUnit.MILLISECONDS.toNanos(config.getFlushMillis()) / 2, 1); LockSupport.parkNanos(HttpPostEmitter.this, waitNanos); } } }
@Test public void testSettingEverything() { final Properties props = new Properties(); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.flushMillis", "1"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.flushCount", "2"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.basicAuthentication", "a:b"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.batchingStrategy", "NEWLINES"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.maxBatchSize", "4"); props.setProperty("org.apache.druid.java.util.emitter.httpEmitting.flushTimeOut", "1000"); final ObjectMapper objectMapper = new ObjectMapper(); final ParametrizedUriEmitterConfig paramConfig = objectMapper.convertValue(Emitters.makeCustomFactoryMap(props), ParametrizedUriEmitterConfig.class); final HttpEmitterConfig config = paramConfig.buildHttpEmitterConfig("http://example.com/topic"); Assert.assertEquals(1, config.getFlushMillis()); Assert.assertEquals(2, config.getFlushCount()); Assert.assertEquals("http://example.com/topic", config.getRecipientBaseUrl()); Assert.assertEquals("a:b", config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.NEWLINES, config.getBatchingStrategy()); Assert.assertEquals(4, config.getMaxBatchSize()); Assert.assertEquals(1000, config.getFlushTimeOut()); } }
@Test public void testSettingEverythingLegacy() { final Properties props = new Properties(); props.setProperty("org.apache.druid.java.util.emitter.flushMillis", "1"); props.setProperty("org.apache.druid.java.util.emitter.flushCount", "2"); props.setProperty("org.apache.druid.java.util.emitter.http.url", "http://example.com/"); props.setProperty("org.apache.druid.java.util.emitter.http.basicAuthentication", "a:b"); props.setProperty("org.apache.druid.java.util.emitter.http.batchingStrategy", "newlines"); props.setProperty("org.apache.druid.java.util.emitter.http.maxBatchSize", "4"); props.setProperty("org.apache.druid.java.util.emitter.http.flushTimeOut", "1000"); props.setProperty("org.apache.druid.java.util.emitter.http.batchQueueSizeLimit", "2500"); props.setProperty("org.apache.druid.java.util.emitter.http.httpTimeoutAllowanceFactor", "3.0"); props.setProperty("org.apache.druid.java.util.emitter.http.minHttpTimeoutMillis", "100"); final ObjectMapper objectMapper = new ObjectMapper(); final HttpEmitterConfig config = objectMapper.convertValue(Emitters.makeHttpMap(props), HttpEmitterConfig.class); Assert.assertEquals(1, config.getFlushMillis()); Assert.assertEquals(2, config.getFlushCount()); Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals("a:b", config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.NEWLINES, config.getBatchingStrategy()); Assert.assertEquals(4, config.getMaxBatchSize()); Assert.assertEquals(1000, config.getFlushTimeOut()); Assert.assertEquals(2500, config.getBatchQueueSizeLimit()); Assert.assertEquals(3.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(100, config.getMinHttpTimeoutMillis()); }
@Test public void testSettingEverything() { final Properties props = new Properties(); props.setProperty("org.apache.druid.java.util.emitter.flushMillis", "1"); props.setProperty("org.apache.druid.java.util.emitter.flushCount", "2"); props.setProperty("org.apache.druid.java.util.emitter.recipientBaseUrl", "http://example.com/"); props.setProperty("org.apache.druid.java.util.emitter.basicAuthentication", "a:b"); props.setProperty("org.apache.druid.java.util.emitter.batchingStrategy", "NEWLINES"); props.setProperty("org.apache.druid.java.util.emitter.maxBatchSize", "4"); props.setProperty("org.apache.druid.java.util.emitter.flushTimeOut", "1000"); props.setProperty("org.apache.druid.java.util.emitter.batchQueueSizeLimit", "2500"); props.setProperty("org.apache.druid.java.util.emitter.httpTimeoutAllowanceFactor", "3.0"); props.setProperty("org.apache.druid.java.util.emitter.minHttpTimeoutMillis", "100"); final ObjectMapper objectMapper = new ObjectMapper(); final HttpEmitterConfig config = objectMapper.convertValue( Emitters.makeCustomFactoryMap(props), HttpEmitterConfig.class ); Assert.assertEquals(1, config.getFlushMillis()); Assert.assertEquals(2, config.getFlushCount()); Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals("a:b", config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.NEWLINES, config.getBatchingStrategy()); Assert.assertEquals(4, config.getMaxBatchSize()); Assert.assertEquals(1000, config.getFlushTimeOut()); Assert.assertEquals(2500, config.getBatchQueueSizeLimit()); Assert.assertEquals(3.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(100, config.getMinHttpTimeoutMillis()); }
@Test public void testDefaults() { final Properties props = new Properties(); final ObjectMapper objectMapper = new ObjectMapper(); final ParametrizedUriEmitterConfig paramConfig = objectMapper.convertValue(Emitters.makeCustomFactoryMap(props), ParametrizedUriEmitterConfig.class); final HttpEmitterConfig config = paramConfig.buildHttpEmitterConfig("http://example.com/topic"); Assert.assertEquals(60000, config.getFlushMillis()); Assert.assertEquals(500, config.getFlushCount()); Assert.assertEquals("http://example.com/topic", config.getRecipientBaseUrl()); Assert.assertEquals(null, config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); Pair<Integer, Integer> batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( Runtime.getRuntime().maxMemory() ); Assert.assertEquals(batchConfigPair.lhs.intValue(), config.getMaxBatchSize()); Assert.assertEquals(batchConfigPair.rhs.intValue(), config.getBatchQueueSizeLimit()); Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); }
@Test public void testDefaultsLegacy() { final Properties props = new Properties(); props.put("org.apache.druid.java.util.emitter.http.url", "http://example.com/"); final ObjectMapper objectMapper = new ObjectMapper(); final HttpEmitterConfig config = objectMapper.convertValue(Emitters.makeHttpMap(props), HttpEmitterConfig.class); Assert.assertEquals(60000, config.getFlushMillis()); Assert.assertEquals(300, config.getFlushCount()); Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals(null, config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); Pair<Integer, Integer> batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( Runtime.getRuntime().maxMemory() ); Assert.assertEquals(batchConfigPair.lhs.intValue(), config.getMaxBatchSize()); Assert.assertEquals(batchConfigPair.rhs.intValue(), config.getBatchQueueSizeLimit()); Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); Assert.assertEquals(2.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(0, config.getMinHttpTimeoutMillis()); }
@Test public void testDefaults() { final Properties props = new Properties(); props.put("org.apache.druid.java.util.emitter.recipientBaseUrl", "http://example.com/"); final ObjectMapper objectMapper = new ObjectMapper(); final HttpEmitterConfig config = objectMapper.convertValue( Emitters.makeCustomFactoryMap(props), HttpEmitterConfig.class ); Assert.assertEquals(60000, config.getFlushMillis()); Assert.assertEquals(500, config.getFlushCount()); Assert.assertEquals("http://example.com/", config.getRecipientBaseUrl()); Assert.assertEquals(null, config.getBasicAuthentication()); Assert.assertEquals(BatchingStrategy.ARRAY, config.getBatchingStrategy()); Pair<Integer, Integer> batchConfigPair = BaseHttpEmittingConfig.getDefaultBatchSizeAndLimit( Runtime.getRuntime().maxMemory() ); Assert.assertEquals(batchConfigPair.lhs.intValue(), config.getMaxBatchSize()); Assert.assertEquals(batchConfigPair.rhs.intValue(), config.getBatchQueueSizeLimit()); Assert.assertEquals(Long.MAX_VALUE, config.getFlushTimeOut()); Assert.assertEquals(2.0f, config.getHttpTimeoutAllowanceFactor(), 0.0f); Assert.assertEquals(0, config.getMinHttpTimeoutMillis()); }
void sealIfFlushNeeded() { long timeSinceFirstEvent = System.currentTimeMillis() - firstEventTimestamp; if (firstEventTimestamp > 0 && timeSinceFirstEvent > emitter.config.getFlushMillis()) { seal(); } }
private void unlockAndSealIfNeeded() { if (eventCount.incrementAndGet() >= emitter.config.getFlushCount()) { unlockAndSeal(); } else { long timeSinceFirstEvent = System.currentTimeMillis() - firstEventTimestamp; if (firstEventTimestamp > 0 && timeSinceFirstEvent > emitter.config.getFlushMillis()) { unlockAndSeal(); } else { unlock(); } } }
@Override public void run() { while (true) { boolean needsToShutdown = needsToShutdown(); try { emitLargeEvents(); emitBatches(); tryEmitOneFailedBuffer(); if (needsToShutdown) { tryEmitAndDrainAllFailedBuffers(); // Make GC life easier drainBuffersToReuse(); return; } } catch (Throwable t) { log.error(t, "Uncaught exception in EmittingThread.run()"); } if (failedBuffers.isEmpty()) { // Waiting for 1/2 of config.getFlushMillis() in order to flush events not more than 50% later than specified. // If nanos=0 parkNanos() doesn't wait at all, then we don't want. long waitNanos = Math.max(TimeUnit.MILLISECONDS.toNanos(config.getFlushMillis()) / 2, 1); LockSupport.parkNanos(HttpPostEmitter.this, waitNanos); } } }