@Override public void run() { try { // Its possible that while the notification was in the queue the situation changed and it // should no longer be processed by this worker. So ask as late as possible if this // notification should be processed. if (notificationFinder.shouldProcess(notification)) { workTask.run(); } } catch (Exception e) { log.error("Failed to process work " + Hex.encNonAscii(notification), e); } finally { tracker.remove(notification); workFinished(); } } };
int qSize = hwf.getWorkerQueue().size(); while (hwf.getWorkerQueue().size() > qSize / 2 && !stopped.get()) { UtilWaitThread.sleep(50, stopped); List<TabletInfo<TabletData>> tablets = new ArrayList<>(tabletInfoCache.getTablets()); Collections.shuffle(tablets, rand); try { for (TabletInfo<TabletData> tabletInfo : tablets) { if (System.currentTimeMillis() >= tabletInfo.getData().retryTime) { int count = 0; ModulusParams modParams = hwf.getModulusParams(); if (modParams != null) { count = scan(modParams, tabletInfo.getRange()); tabletsScanned++; tabletInfo.getData().updateScanCount(count, maxSleepTime); notifications += count; if (stopped.get()) { minRetryTime = Math.min(tabletInfo.getData().retryTime, minRetryTime); hwf.getWorkerQueue().clear(); waitForFindersToStabilize(); qSize = hwf.getWorkerQueue().size();
env.getConfiguration().print(); NotificationProcessor np = new NotificationProcessor(env); NotificationFinder notificationFinder = NotificationFinderFactory.newNotificationFinder(env.getConfiguration()); notificationFinder.init(env, np); notificationFinder.start(); notificationFinder.stop();
public MiniFluoImpl(FluoConfiguration fluoConfig) { if (!fluoConfig.hasRequiredMiniFluoProps()) { throw new IllegalArgumentException("MiniFluo configuration is not valid"); } config = fluoConfig; try { if (config.getMiniStartAccumulo()) { startMiniAccumulo(); } config.setProperty(FluoConfigurationImpl.MIN_SLEEP_TIME_PROP, 50); config.setProperty(FluoConfigurationImpl.MAX_SLEEP_TIME_PROP, 100); env = new Environment(config); reporter = FluoClientImpl.setupReporters(env, "mini", reporterCounter); oserver = new OracleServer(env); oserver.start(); mnp = new MiniNotificationProcessor(env); notificationFinder = NotificationFinderFactory.newNotificationFinder(env.getConfiguration()); notificationFinder.init(env, mnp); notificationFinder.start(); } catch (Exception e) { throw new RuntimeException(e); } }
public boolean addNotification(final NotificationFinder notificationFinder, final Notification notification) { final WorkTask workTask = new WorkTask(notificationFinder, env, notification, observers); if (!tracker.add(notification, ft)) { return false; workAdded(); tracker.remove(notification); workFinished(); throw rje;
@Override public void run() { Observer observer = observers.getObserver(notification.getColumn()); try { int sleepTime = 0; } catch (AlreadyAcknowledgedException aae) { status = TxResult.AACKED; notificationFinder.failedToProcess(notification, status); break; } catch (CommitException e) { log.warn("Failed to execute observer " + observer.getClass().getSimpleName() + " notification : " + Hex.encNonAscii(notification), e); notificationFinder.failedToProcess(notification, status); break; } finally { if (txi != null) { try { txi.getStats().report(status.toString(), observer.getClass()); } finally { tx.close(); observers.returnObserver(observer);
private int scan(ModulusParams lmp, Range range) throws TableNotFoundException { Scanner scanner = env.getConnector().createScanner(env.getTable(), env.getAuthorizations()); scanner.setRange(range); Notification.configureScanner(scanner); IteratorSetting iterCfg = new IteratorSetting(30, "nhf", NotificationHashFilter.class); NotificationHashFilter.setModulusParams(iterCfg, lmp.divisor, lmp.remainder); scanner.addScanIterator(iterCfg); int count = 0; for (Entry<Key, Value> entry : scanner) { if (lmp.update != hwf.getModulusParams().update) { throw new HashNotificationFinder.ModParamsChangedException(); } if (stopped.get()) { return count; } if (hwf.getWorkerQueue().addNotification(hwf, Notification.from(entry.getKey()))) { count++; } } return count; }
public NotificationProcessor(Environment env) { int numThreads = env.getConfiguration().getWorkerThreads(); this.env = env; this.queue = new LinkedBlockingQueue<>(); this.executor = new ThreadPoolExecutor(numThreads, numThreads, 0L, TimeUnit.MILLISECONDS, queue); this.tracker = new NotificationTracker(); this.observers = new Observers(env); env.getSharedResources().getMetricRegistry() .register(env.getMetricNames().getNotificationQueued(), new Gauge<Integer>() { @Override public Integer getValue() { return queue.size(); } }); }
@Override public void close() { try { if (oserver != null) { notificationFinder.stop(); mnp.close(); oserver.stop(); env.close(); reporter.close(); if (cluster != null) { cluster.stop(); } } } catch (Exception e) { throw new RuntimeException(e); } }
public Observer getObserver(Column col) { observerList = getObserverList(col); observer = Class.forName(observerConfig.getClassName()).asSubclass(Observer.class).newInstance(); observer.init(new ObserverContext(env, observerConfig.getParameters())); } catch (RuntimeException e) { throw e;
public synchronized void remove(RowColumn rowCol) { if (queuedWork.remove(rowCol) != null) { sizeInBytes -= size(rowCol); notify(); } }
@Override public void close() { executor.shutdownNow(); observers.close(); try { while (!executor.awaitTermination(1, TimeUnit.SECONDS)) { } } catch (InterruptedException e) { throw new RuntimeException(e); } }
public void clear() { tracker.clear(); executor.purge(); }
public void returnObserver(Observer observer) { List<Observer> olist = getObserverList(observer.getObservedColumn().getColumn()); synchronized (olist) { olist.add(observer); } }
public synchronized List<TabletInfo<T>> getTablets() throws Exception { if (cachedTablets == null) { cachedTablets = listSplits(); } else if (System.currentTimeMillis() - listSplitsTime > CACHE_TIME) { List<TabletInfo<T>> tablets = listSplits(); Map<TabletInfo<T>, TabletInfo<T>> oldTablets = new HashMap<>(); for (TabletInfo<T> tabletInfo : cachedTablets) { oldTablets.put(tabletInfo, tabletInfo); } List<TabletInfo<T>> newTablets = new ArrayList<>(tablets.size()); for (TabletInfo<T> tabletInfo : tablets) { TabletInfo<T> oldTI = oldTablets.get(tabletInfo); if (oldTI != null) { newTablets.add(oldTI); } else { newTablets.add(tabletInfo); } } cachedTablets = newTablets; } return Collections.unmodifiableList(cachedTablets); } }
private List<TabletInfo<T>> listSplits() throws TableNotFoundException, AccumuloSecurityException, AccumuloException { List<Text> splits = new ArrayList<>(env.getConnector().tableOperations().listSplits(env.getTable())); Collections.sort(splits); List<TabletInfo<T>> tablets = new ArrayList<>(splits.size() + 1); for (int i = 0; i < splits.size(); i++) { tablets .add(new TabletInfo<>(i == 0 ? null : splits.get(i - 1), splits.get(i), supplier.get())); } tablets.add(new TabletInfo<>(splits.size() == 0 ? null : splits.get(splits.size() - 1), null, supplier.get())); listSplitsTime = System.currentTimeMillis(); return tablets; }
ScanTask(HashNotificationFinder hashWorkFinder, Environment env, AtomicBoolean stopped) { this.hwf = hashWorkFinder; this.tabletInfoCache = new TabletInfoCache<TabletData, Supplier<TabletData>>(env, new Supplier<TabletData>() { @Override public TabletData get() { return new TabletData(); } }); this.env = env; this.stopped = stopped; minSleepTime = env.getConfiguration().getInt(FluoConfigurationImpl.MIN_SLEEP_TIME_PROP, FluoConfigurationImpl.MIN_SLEEP_TIME_DEFAULT); maxSleepTime = env.getConfiguration().getInt(FluoConfigurationImpl.MAX_SLEEP_TIME_PROP, FluoConfigurationImpl.MAX_SLEEP_TIME_DEFAULT); }
public synchronized boolean add(RowColumn rowCol, Future<?> task) { if (queuedWork.containsKey(rowCol)) { return false; } while (sizeInBytes > MAX_SIZE) { try { wait(1000); } catch (InterruptedException e) { throw new RuntimeException(e); } } if (queuedWork.containsKey(rowCol)) { return false; } queuedWork.put(rowCol, task); sizeInBytes += size(rowCol); return true; }