/** * Persist all data indexed through this driver so far. Returns a future of persisted commitMetadata. * <p> * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. * * @param committer committer representing all data that has been added so far * * @return future containing commitMetadata persisted */ public ListenableFuture<Object> persistAsync(final Committer committer) { return appenderator.persistAll(wrapCommitter(committer)); }
@Override public void persist(final Committer committer) { final Stopwatch runExecStopwatch = Stopwatch.createStarted(); appenderator.persistAll(committer); final long startDelay = runExecStopwatch.elapsed(TimeUnit.MILLISECONDS); metrics.incrementPersistBackPressureMillis(startDelay); if (startDelay > WARN_DELAY) { log.warn("Ingestion was throttled for [%,d] millis because persists were pending.", startDelay); } runExecStopwatch.stop(); }
/** * Persist all data indexed through this driver so far. Blocks until complete. * <p> * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. * * @param committer committer representing all data that has been added so far * * @return commitMetadata persisted */ public Object persist(final Committer committer) throws InterruptedException { try { log.info("Persisting data."); final long start = System.currentTimeMillis(); final Object commitMetadata = appenderator.persistAll(wrapCommitter(committer)).get(); log.info("Persisted pending data in %,dms.", System.currentTimeMillis() - start); return commitMetadata; } catch (InterruptedException e) { throw e; } catch (Exception e) { throw new RuntimeException(e); } }
Assert.assertEquals(2, appenderator.getTotalRowCount()); appenderator.persistAll(committerSupplier.get()).get(); Assert.assertEquals(2, appenderator.getTotalRowCount()); appenderator.drop(IDENTIFIERS.get(0)).get(); Assert.assertEquals(4, appenderator.getTotalRowCount()); appenderator.persistAll(committerSupplier.get()).get(); Assert.assertEquals(4, appenderator.getTotalRowCount()); appenderator.drop(IDENTIFIERS.get(2)).get();
appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier, false); Assert.assertEquals(5, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.persistAll(committerSupplier.get()); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.close();
appenderator.add(IDENTIFIERS.get(0), IR("2000", "bob", 1), committerSupplier); Assert.assertEquals(2, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.persistAll(committerSupplier.get()); Assert.assertEquals(0, ((AppenderatorImpl) appenderator).getRowsInMemory()); appenderator.close();
/** * Persist all data indexed through this driver so far. Returns a future of persisted commitMetadata. * * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. * * @param committer committer representing all data that has been added so far * * @return future containing commitMetadata persisted */ public ListenableFuture<Object> persistAsync(final Committer committer) { return appenderator.persistAll(wrapCommitter(committer)); }
@Override public void persist(final Committer committer) { final Stopwatch runExecStopwatch = Stopwatch.createStarted(); appenderator.persistAll(committer); final long startDelay = runExecStopwatch.elapsed(TimeUnit.MILLISECONDS); metrics.incrementPersistBackPressureMillis(startDelay); if (startDelay > WARN_DELAY) { log.warn("Ingestion was throttled for [%,d] millis because persists were pending.", startDelay); } runExecStopwatch.stop(); }
/** * Persist all data indexed through this driver so far. Blocks until complete. * * Should be called after all data has been added through {@link #add(InputRow, String, Supplier, boolean, boolean)}. * * @param committer committer representing all data that has been added so far * * @return commitMetadata persisted */ public Object persist(final Committer committer) throws InterruptedException { try { log.info("Persisting data."); final long start = System.currentTimeMillis(); final Object commitMetadata = appenderator.persistAll(wrapCommitter(committer)).get(); log.info("Persisted pending data in %,dms.", System.currentTimeMillis() - start); return commitMetadata; } catch (InterruptedException e) { throw e; } catch (Exception e) { throw new RuntimeException(e); } }