CircularBuffer<Integer> circularBuffer = new CircularBuffer<>( 3); circularBuffer.add(1); Assert.assertEquals(1, circularBuffer.size()); Assert.assertEquals(1, (int) circularBuffer.get(0)); circularBuffer.add(2); Assert.assertEquals(2, circularBuffer.size()); for (int i = 0; i < circularBuffer.size(); i++) { Assert.assertEquals(i + 1, (int) circularBuffer.get(i)); circularBuffer.add(3); Assert.assertEquals(3, circularBuffer.size()); for (int i = 0; i < circularBuffer.size(); i++) { Assert.assertEquals(i + 1, (int) circularBuffer.get(i)); circularBuffer.add(4); Assert.assertEquals(3, circularBuffer.size()); for (int i = 0; i < circularBuffer.size(); i++) { Assert.assertEquals(i + 2, (int) circularBuffer.get(i)); circularBuffer.add(5); Assert.assertEquals(3, circularBuffer.size()); for (int i = 0; i < circularBuffer.size(); i++) { Assert.assertEquals(i + 3, (int) circularBuffer.get(i)); circularBuffer.add(6); Assert.assertEquals(3, circularBuffer.size());
@Nullable public static List<String> getMessagesFromSavedParseExceptions(CircularBuffer<Throwable> savedParseExceptions) { if (savedParseExceptions == null) { return null; } List<String> events = new ArrayList<>(); for (int i = 0; i < savedParseExceptions.size(); i++) { events.add(savedParseExceptions.getLatest(i).getMessage()); } return events; }
/** * Add batch of segment changes update. */ public synchronized void addChangeRequests(List<T> requests) { for (T request : requests) { changes.add(new Holder<>(request, getLastCounter().inc())); } singleThreadedExecutor.execute(resolveWaitingFuturesRunnable); }
public ChangeRequestHistory(int maxSize) { this.maxSize = maxSize; this.changes = new CircularBuffer<>(maxSize); this.waitingFutures = new LinkedHashMap<>(); this.resolveWaitingFuturesRunnable = this::resolveWaitingFutures; this.singleThreadedExecutor = Execs.singleThreaded("SegmentChangeRequestHistory"); }
private void handleParseException(ParseException e) { if (e.isFromPartiallyValidRow()) { buildSegmentsMeters.incrementProcessedWithError(); } else { buildSegmentsMeters.incrementUnparseable(); } if (ingestionSchema.tuningConfig.isLogParseExceptions()) { log.error(e, "Encountered parse exception:"); } if (buildSegmentsSavedParseExceptions != null) { buildSegmentsSavedParseExceptions.add(e); } if (buildSegmentsMeters.getUnparseable() + buildSegmentsMeters.getProcessedWithError() > ingestionSchema.tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task...", e); } }
int changeStartIndex = (int) (counter.counter + changes.size() - lastCounter.counter); Counter counterToMatch = counter.counter == 0 ? Counter.ZERO : changes.get(changeStartIndex - 1).counter; if (!counterToMatch.matches(counter)) { return ChangeRequestsSnapshot.fail( for (int i = changeStartIndex; i < changes.size(); i++) { result.add(changes.get(i).changeRequest); return ChangeRequestsSnapshot.success(changes.get(changes.size() - 1).counter, result);
this.chatHandlerProvider = Optional.fromNullable(chatHandlerProvider); if (tuningConfig.getMaxSavedParseExceptions() > 0) { savedParseExceptions = new CircularBuffer<>(tuningConfig.getMaxSavedParseExceptions()); } else { savedParseExceptions = null;
determinePartitionsSavedParseExceptions.add(e);
@JsonCreator public AppenderatorDriverRealtimeIndexTask( @JsonProperty("id") String id, @JsonProperty("resource") TaskResource taskResource, @JsonProperty("spec") RealtimeAppenderatorIngestionSpec spec, @JsonProperty("context") Map<String, Object> context, @JacksonInject ChatHandlerProvider chatHandlerProvider, @JacksonInject AuthorizerMapper authorizerMapper, @JacksonInject RowIngestionMetersFactory rowIngestionMetersFactory ) { super( id == null ? makeTaskId(spec) : id, StringUtils.format("index_realtime_appenderator_%s", spec.getDataSchema().getDataSource()), taskResource, spec.getDataSchema().getDataSource(), context ); this.spec = spec; this.pendingHandoffs = new ConcurrentLinkedQueue<>(); this.chatHandlerProvider = Optional.fromNullable(chatHandlerProvider); this.authorizerMapper = authorizerMapper; if (spec.getTuningConfig().getMaxSavedParseExceptions() > 0) { savedParseExceptions = new CircularBuffer<>(spec.getTuningConfig().getMaxSavedParseExceptions()); } this.ingestionState = IngestionState.NOT_STARTED; this.rowIngestionMeters = rowIngestionMetersFactory.createRowIngestionMeters(); }
@Nullable public static List<String> getMessagesFromSavedParseExceptions(CircularBuffer<Throwable> savedParseExceptions) { if (savedParseExceptions == null) { return null; } List<String> events = new ArrayList<>(); for (int i = 0; i < savedParseExceptions.size(); i++) { events.add(savedParseExceptions.getLatest(i).getMessage()); } return events; }
private void handleParseException(ParseException pe) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (spec.getTuningConfig().isLogParseExceptions()) { log.error(pe, "Encountered parse exception: "); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > spec.getTuningConfig().getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
int changeStartIndex = (int) (counter.counter + changes.size() - lastCounter.counter); Counter counterToMatch = counter.counter == 0 ? Counter.ZERO : changes.get(changeStartIndex - 1).counter; if (!counterToMatch.matches(counter)) { return ChangeRequestsSnapshot.fail( for (int i = changeStartIndex; i < changes.size(); i++) { result.add(changes.get(i).changeRequest); return ChangeRequestsSnapshot.success(changes.get(changes.size() - 1).counter, result);
this.chatHandlerProvider = Optional.fromNullable(chatHandlerProvider); if (ingestionSchema.getTuningConfig().getMaxSavedParseExceptions() > 0) { determinePartitionsSavedParseExceptions = new CircularBuffer<Throwable>( ingestionSchema.getTuningConfig().getMaxSavedParseExceptions() ); buildSegmentsSavedParseExceptions = new CircularBuffer<Throwable>( ingestionSchema.getTuningConfig().getMaxSavedParseExceptions() );
private void handleParseException(ParseException pe, ConsumerRecord<byte[], byte[]> record) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (tuningConfig.isLogParseExceptions()) { log.error( pe, "Encountered parse exception on row from partition[%d] offset[%d]", record.partition(), record.offset() ); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
public ChangeRequestHistory(int maxSize) { this.maxSize = maxSize; this.changes = new CircularBuffer(maxSize); this.waitingFutures = new LinkedHashMap<>(); this.resolveWaitingFuturesRunnable = new Runnable() { @Override public void run() { resolveWaitingFutures(); } }; this.singleThreadedExecutor = Executors.newSingleThreadExecutor( new ThreadFactoryBuilder().setDaemon(true) .setNameFormat( "SegmentChangeRequestHistory" ) .build() ); }
private void handleParseException(ParseException pe, OrderedPartitionableRecord record) { if (pe.isFromPartiallyValidRow()) { rowIngestionMeters.incrementProcessedWithError(); } else { rowIngestionMeters.incrementUnparseable(); } if (tuningConfig.isLogParseExceptions()) { log.error( pe, "Encountered parse exception on row from partition[%s] sequence[%s]", record.getPartitionId(), record.getSequenceNumber() ); } if (savedParseExceptions != null) { savedParseExceptions.add(pe); } if (rowIngestionMeters.getUnparseable() + rowIngestionMeters.getProcessedWithError() > tuningConfig.getMaxParseExceptions()) { log.error("Max parse exceptions exceeded, terminating task..."); throw new RuntimeException("Max parse exceptions exceeded, terminating task..."); } }
@JsonCreator public AppenderatorDriverRealtimeIndexTask( @JsonProperty("id") String id, @JsonProperty("resource") TaskResource taskResource, @JsonProperty("spec") RealtimeAppenderatorIngestionSpec spec, @JsonProperty("context") Map<String, Object> context, @JacksonInject ChatHandlerProvider chatHandlerProvider, @JacksonInject AuthorizerMapper authorizerMapper, @JacksonInject RowIngestionMetersFactory rowIngestionMetersFactory ) { super( id == null ? makeTaskId(spec) : id, StringUtils.format("index_realtime_appenderator_%s", spec.getDataSchema().getDataSource()), taskResource, spec.getDataSchema().getDataSource(), context ); this.spec = spec; this.pendingHandoffs = new ConcurrentLinkedQueue<>(); this.chatHandlerProvider = Optional.fromNullable(chatHandlerProvider); this.authorizerMapper = authorizerMapper; if (spec.getTuningConfig().getMaxSavedParseExceptions() > 0) { savedParseExceptions = new CircularBuffer<>(spec.getTuningConfig().getMaxSavedParseExceptions()); } this.ingestionState = IngestionState.NOT_STARTED; this.rowIngestionMeters = rowIngestionMetersFactory.createRowIngestionMeters(); }
/** * Add batch of segment changes update. */ public synchronized void addChangeRequests(List<T> requests) { for (T request : requests) { changes.add(new Holder(request, getLastCounter().inc())); } singleThreadedExecutor.execute(resolveWaitingFuturesRunnable); }