private void incrementCounter(K counter) { emitted++; MutableLong ml = counters.get(counter); if (ml == null) { if (counters.size() >= maxCounters) { // no need to store this counter in the map and get() it... just use instance variable tooMany++; } else { // we have never seen this key before, check if its too long if (encoder.apply(counter).length() >= maxCounterKeyLen) { tooLong++; } else { counters.put(copier.apply(counter), new MutableLong(1)); } } } else { // using mutable long allows calling put() to be avoided ml.increment(); } }
private void convertMutations(TabletServerMutations<QCMutation> mutations, Map<Long,CMK> cmidToCm, MutableLong cmid, Map<TKeyExtent,List<TConditionalMutation>> tmutations, CompressedIterators compressedIters) { for (Entry<KeyExtent,List<QCMutation>> entry : mutations.getMutations().entrySet()) { TKeyExtent tke = entry.getKey().toThrift(); ArrayList<TConditionalMutation> tcondMutaions = new ArrayList<>(); List<QCMutation> condMutations = entry.getValue(); for (QCMutation cm : condMutations) { TMutation tm = cm.toThrift(); List<TCondition> conditions = convertConditions(cm, compressedIters); cmidToCm.put(cmid.longValue(), new CMK(entry.getKey(), cm)); TConditionalMutation tcm = new TConditionalMutation(conditions, tm, cmid.longValue()); cmid.increment(); tcondMutaions.add(tcm); } tmutations.put(tke, tcondMutaions); } }
count.increment();
/** * Helper method for handling IOExceptions. * @param e The caught IOException. */ private void failureHandling(Exception e) { localNumberOfFailures.increment(); if (maxRetryCount <= 0) { throw new RuntimeException(e); } LOG.error("FS reader error", e); addToFailedList(); }
protected void addToFailedList() { FailedFile ff = new FailedFile(currentFile, offset, retryCount); try { // try to close file if (this.inputStream != null) { this.inputStream.close(); } } catch (IOException e) { localNumberOfFailures.increment(); LOG.error("Could not close input stream on: " + currentFile); } ff.retryCount++; ff.lastFailedTime = System.currentTimeMillis(); ff.offset = this.offset; // Clear current file state. this.currentFile = null; this.inputStream = null; if (ff.retryCount > maxRetryCount) { return; } localNumberOfRetries.increment(); LOG.info("adding to failed list path {} offset {} retry {}", ff.path, ff.offset, ff.retryCount); failedFiles.add(ff); }
@Override public void process(Map<String,String> tuple) { LOG.info("new query {}", tuple); String command = tuple.get(KEY_COMMAND); if (command != null) { if (command.equals(COMMAND_ADD)) { commandCounters.getCounter(CommandCounters.ADD).increment(); String phoneStr = tuple.get(KEY_PHONE); registerPhone(phoneStr); } else if (command.equals(COMMAND_ADD_RANGE)) { commandCounters.getCounter(CommandCounters.ADD_RANGE).increment(); registerPhoneRange(tuple.get(KEY_START_PHONE), tuple.get(KEY_END_PHONE)); } else if (command.equals(COMMAND_DELETE)) { commandCounters.getCounter(CommandCounters.DELETE).increment(); String phoneStr = tuple.get(KEY_PHONE); deregisterPhone(phoneStr); } else if (command.equals(COMMAND_CLEAR)) { commandCounters.getCounter(CommandCounters.CLEAR).increment(); clearPhones(); } } } };
/** * Scans the directory for new files. */ protected void scanDirectory() { if (System.currentTimeMillis() - scanIntervalMillis >= lastScanMillis) { Set<Path> newPaths = scanner.scan(fs, filePath, processedFiles); for (Path newPath : newPaths) { try { FileStatus fileStatus = fs.getFileStatus(newPath); if (fileStatus.isDirectory()) { checkVisitedDirectory(newPath); } else { String newPathString = newPath.toString(); pendingFiles.add(newPathString); processedFiles.add(newPathString); localProcessedFileCount.increment(); } } catch (IOException e) { throw new RuntimeException(e); } } lastScanMillis = System.currentTimeMillis(); } }
private void processRecord(byte[] key, MVCCRecord record, List<byte[]> resultKeys, List<MVCCRecord> resultValues, MutableLong numKvs, RangeOption<K> rangeOption, boolean countOnly) { if (null == rangeOption && countOnly) { numKvs.increment(); return; } if (record.test(rangeOption)) { numKvs.increment(); if (countOnly) { record.recycle(); } else { resultKeys.add(key); resultValues.add(record); } } else { record.recycle(); } }
@Override public void endWindow() { if (currentWindowId > windowDataManager.getLargestCompletedWindow()) { try { windowDataManager.save(currentWindowRecoveryState, currentWindowId); } catch (IOException e) { throw new RuntimeException("saving recovery", e); } } currentWindowRecoveryState.clear(); if (context != null) { pendingFileCount.setValue(pendingFiles.size() + failedFiles.size() + unfinishedFiles.size()); if (currentFile != null) { pendingFileCount.increment(); } context.setCounters(fileCounters); } }
private void convertMutations(TabletServerMutations<QCMutation> mutations, Map<Long,CMK> cmidToCm, MutableLong cmid, Map<TKeyExtent,List<TConditionalMutation>> tmutations, CompressedIterators compressedIters) { for (Entry<KeyExtent,List<QCMutation>> entry : mutations.getMutations().entrySet()) { TKeyExtent tke = entry.getKey().toThrift(); ArrayList<TConditionalMutation> tcondMutaions = new ArrayList<>(); List<QCMutation> condMutations = entry.getValue(); for (QCMutation cm : condMutations) { TMutation tm = cm.toThrift(); List<TCondition> conditions = convertConditions(cm, compressedIters); cmidToCm.put(cmid.longValue(), new CMK(entry.getKey(), cm)); TConditionalMutation tcm = new TConditionalMutation(conditions, tm, cmid.longValue()); cmid.increment(); tcondMutaions.add(tcm); } tmutations.put(tke, tcondMutaions); } }
/** * This method is called when a message is added to {@link #holdingBuffer} and can be overwritten by subclasses * if required. This is called by the JMS thread not Operator thread. * * @param message * @return message is accepted. * @throws javax.jms.JMSException */ protected boolean messageConsumed(Message message) throws JMSException { if (message.getJMSRedelivered() && pendingAck.contains(message.getJMSMessageID())) { counters.getCounter(CounterKeys.REDELIVERED).increment(); LOG.warn("IGNORING: Redelivered Message {}", message.getJMSMessageID()); return false; } pendingAck.add(message.getJMSMessageID()); MutableLong receivedCt = counters.getCounter(CounterKeys.RECEIVED); receivedCt.increment(); LOG.debug("message id: {} buffer size: {} received: {}", message.getJMSMessageID(), holdingBuffer.size(), receivedCt.longValue()); return true; }
/** * Adds the values for each key, counts the number of occurrences of each * key and computes the average. */ @Override public void process(KeyValPair<K, ? extends Number> tuple) { K key = tuple.getKey(); if (!doprocessKey(key)) { return; } MutableDouble val = sums.get(key); if (val == null) { val = new MutableDouble(tuple.getValue().doubleValue()); } else { val.add(tuple.getValue().doubleValue()); } sums.put(cloneKey(key), val); MutableLong count = counts.get(key); if (count == null) { count = new MutableLong(0); counts.put(cloneKey(key), count); } count.increment(); } };
fileCounters.getCounter(Counters.PROCESSED_FILES).increment(); filesMetadataOutput.emit(fileMetadata); blockMetadataIterator = new BlockMetadataIterator(this, fileMetadata, blockSize);
FileMetadata fileMetadata = buildFileMetadata(fileInfo); filesMetadataOutput.emit(fileMetadata); fileCounters.getCounter(Counters.PROCESSED_FILES).increment(); if (!fileMetadata.isDirectory()) { blockMetadataIterator = new BlockMetadataIterator(this, fileMetadata, blockSize);
/** * Read the block data and emit records based on reader context * * @param blockMetadata * block * @throws IOException */ protected void readBlock(BlockMetadata blockMetadata) throws IOException { readerContext.initialize(stream, blockMetadata, consecutiveBlock); ReaderContext.Entity entity; while ((entity = readerContext.next()) != null) { counters.getCounter(ReaderCounterKeys.BYTES).add(entity.getUsedBytes()); byte[] record = entity.getRecord(); if (record != null) { counters.getCounter(ReaderCounterKeys.RECORDS).increment(); records.emit(record); } } }
/** * Override this if you want to change how much of the block is read. * * @param blockMetadata block * @throws IOException */ protected void readBlock(BlockMetadata blockMetadata) throws IOException { readerContext.initialize(stream, blockMetadata, consecutiveBlock); ReaderContext.Entity entity; while ((entity = readerContext.next()) != null) { counters.getCounter(ReaderCounterKeys.BYTES).add(entity.getUsedBytes()); bytesRead += entity.getUsedBytes(); R record = convertToRecord(entity.getRecord()); //If the record is partial then ignore the record. if (record != null) { counters.getCounter(ReaderCounterKeys.RECORDS).increment(); messages.emit(new ReaderRecord<>(blockMetadata.getBlockId(), record)); } } }
lep.getEntries((entryId, location) -> { entryLocationIndex.addLocation(batch, ledgerId, entryId, location); numberOfEntries.increment(); return true; });