/** * Handle event. * * @param listeners the listeners * @param evt the evt */ @SuppressWarnings("unchecked") private void handleEvent(List<LensEventListener> listeners, LensEvent evt) { if (listeners != null && !listeners.isEmpty()) { for (LensEventListener listener : listeners) { try { listener.onEvent(evt); } catch (Exception exc) { log.error("Error in handling event {} for listener {}", evt.getEventId(), listener, exc); } } } }
private void notifyEventService(SchedulerAlarmEvent alarmEvent, boolean shouldWait) throws LensException, InterruptedException { LensEventService eventService = LensServices.get().getService(LensEventService.NAME); if (shouldWait) { eventService.notifyEventSync(alarmEvent); } else { eventService.notifyEvent(alarmEvent); } }
private void notifyRerun(SchedulerJobInstanceInfo instanceInfo) throws LensException { getEventService().notifyEvent( new SchedulerAlarmEvent(instanceInfo.getJobId(), new DateTime(instanceInfo.getScheduleTime()), SchedulerAlarmEvent.EventType.SCHEDULE, instanceInfo.getId())); }
protected void notifyEvent(LensEvent event) throws LensException { getEventService().notifyEvent(event); }
@Override public void process(LensEvent event) { processed = true; latch.countDown(); log.info("LensEvent: {}", event.getEventId()); } }
@Override public synchronized void stop() { // unregister LensEventService eventService = LensServices.get().getService(LensEventService.NAME); if (eventService != null) { eventService.removeListener(queryStatusListener); } if (queryStatusListener != null) { queryStatusListener.stop(); } if (reporters != null) { for (ScheduledReporter reporter : reporters) { reporter.stop(); } } log.info("Stopped metrics service"); super.stop(); }
public void start(LensEventService service) { super.start(service); if (service != null) { service.addListenerForType(this, LoggableLensStatistics.class); service.addListenerForType(handler, PartitionEvent.class); rollupHandler.start(service); } else { LOG.warn("Not starting Log Statistics store as event service is not configured"); } }
/** * Create an asynchronous event listener which uses a thread poool to process events. * * @param poolSize size of the event processing pool * @param timeOutSeconds time out in seconds when an idle thread is destroyed * @param isDaemon if the threads used to process should be daemon threads, * if false, then implementation should call stop() * to stop the thread pool */ public AsyncEventListener(int poolSize, long timeOutSeconds, final boolean isDaemon) { eventQueue = new LinkedBlockingQueue<>(); ThreadFactory factory = new BasicThreadFactory.Builder() .namingPattern(getName()+"_AsyncThread-%d") .daemon(isDaemon) .priority(Thread.NORM_PRIORITY) .build(); // fixed pool with min and max equal to poolSize processor = new ThreadPoolExecutor(poolSize, poolSize, timeOutSeconds, TimeUnit.SECONDS, eventQueue, factory); processor.allowCoreThreadTimeOut(true); }
public void stop(LensEventService service) { super.stop(service); if (service != null) { service.removeListenerForType(this, LoggableLensStatistics.class); service.removeListenerForType(handler, PartitionEvent.class); rollupHandler.stop(); } else { LOG.warn("Not stopping Log Statistics store as event service is not configured"); } }
@Override public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { JobDataMap data = jobExecutionContext.getMergedJobDataMap(); DateTime nominalTime = new DateTime(jobExecutionContext.getScheduledFireTime()); SchedulerJobHandle jobHandle = SchedulerJobHandle.fromString(data.getString(JOB_HANDLE)); boolean shouldWait = data.getBoolean(SHOULD_WAIT); SchedulerAlarmEvent alarmEvent = new SchedulerAlarmEvent(jobHandle, nominalTime, SchedulerAlarmEvent.EventType.SCHEDULE, null); try { notifyEventService(alarmEvent, shouldWait); if (jobExecutionContext.getNextFireTime() == null) { SchedulerAlarmEvent expireEvent = (new SchedulerAlarmEvent(jobHandle, nominalTime, SchedulerAlarmEvent.EventType.EXPIRE, null)); notifyEventService(expireEvent, shouldWait); } } catch (LensException e) { log.error("Failed to notify SchedulerAlarmEvent for jobHandle: {} and scheduleTime: {}", jobHandle.getHandleIdString(), nominalTime.toString()); throw new JobExecutionException("Failed to notify alarmEvent", e); } catch (InterruptedException e) { log.error("Job execution tread interrupted", e); } } }
@Test public void testAysncEventListenerPoolThreads(){ AsyncEventListener<QuerySuccess> ayncListener = new DummyAsncEventListener(); for(int i=0; i<10; i++){ try { //A pool thread is created each time an event is submitted until core pool size is reached which is 5 //for this test case. @see org.apache.lens.server.api.events.AsyncEventListener.processor ayncListener.onEvent(null); } catch (LensException e) { assert(false); //Not Expected } } //Verify the core pool Threads after the events have been fired ThreadGroup currentTG = Thread.currentThread().getThreadGroup(); int count = currentTG.activeCount(); Thread[] threads = new Thread[count]; currentTG.enumerate(threads); Set<String> aysncThreadNames = new HashSet<String>(); for(Thread t : threads){ if (t.getName().contains("DummyAsncEventListener_AsyncThread")){ aysncThreadNames.add(t.getName()); } } assertTrue(aysncThreadNames.containsAll(Arrays.asList( "DummyAsncEventListener_AsyncThread-1", "DummyAsncEventListener_AsyncThread-2", "DummyAsncEventListener_AsyncThread-3", "DummyAsncEventListener_AsyncThread-4", "DummyAsncEventListener_AsyncThread-5"))); }
@Override public void run() { while (!Thread.currentThread().isInterrupted()) { // Get all the instance runs which are waiting. List<SchedulerJobInstanceRun> instanceRuns = schedulerDAO.getInstanceRuns(SchedulerJobInstanceState.WAITING); for (SchedulerJobInstanceRun run : instanceRuns) { SchedulerJobInstanceInfo instanceInfo = schedulerDAO.getSchedulerJobInstanceInfo(run.getHandle()); try { run.setInstanceState(run.getInstanceState().nextTransition(ON_PREPARE)); run.setEndTime(System.currentTimeMillis()); schedulerDAO.updateJobInstanceRun(run); eventService.notifyEvent( new SchedulerAlarmEvent(instanceInfo.getJobId(), new DateTime(instanceInfo.getScheduleTime()), SchedulerAlarmEvent.EventType.SCHEDULE, instanceInfo.getId())); } catch (LensException e) { log.error("Not able to notify schedule event for job {} and instanceId {}", instanceInfo.getJobId(), instanceInfo.getId()); } catch (InvalidStateTransitionException e) { log.error("Wrong state transition", e); } } } try { Thread.sleep(delay); } catch (InterruptedException e) { log.warn("Thread WaitingInstanceHandler was inerrupted", e); } } }
/** * Accept. * * @param query the query * @param conf the conf * @param submitOp the submit op * @throws LensException the lens exception */ private void accept(String query, Configuration conf, SubmitOp submitOp) throws LensException { // run through all the query acceptors, and throw Exception if any of them // return false for (QueryAcceptor acceptor : queryAcceptors) { String cause = ""; String rejectionCause = acceptor.accept(query, conf, submitOp); if (rejectionCause != null) { getEventService().notifyEvent(new QueryRejected(System.currentTimeMillis(), query, rejectionCause, null)); throw new BadRequestException("Query not accepted because " + cause); } } getEventService().notifyEvent(new QueryAccepted(System.currentTimeMillis(), null, query, null)); }
@Override public void stop() { if (eventHandlerPool != null) { List<Runnable> pending = eventHandlerPool.shutdownNow(); if (pending != null && !pending.isEmpty()) { StringBuilder pendingMsg = new StringBuilder("Pending Events:"); for (Runnable handler : pending) { if (handler instanceof EventHandler) { pendingMsg.append(((EventHandler) handler).event.getEventId()).append(","); } } log.info("Event listener service stopped while {} events still pending", pending.size()); log.info(pendingMsg.toString()); } } log.info("Event service stopped"); super.stop(); }
private void initializeListeners() { if (conf.getBoolean(QUERY_STATE_LOGGER_ENABLED, true)) { getEventService().addListenerForType(new QueryStatusLogger(), StatusChange.class); log.info("Registered query state logger"); } // Add result formatter getEventService().addListenerForType(new ResultFormatter(this, this.logSegregationContext), QueryExecuted.class); getEventService().addListenerForType(new QueryExecutionStatisticsGenerator(getEventService()), QueryEnded.class); getEventService().addListenerForType( new QueryEndNotifier(this, getCliService().getHiveConf(), this.logSegregationContext), QueryEnded.class); getEventService().addListenerForType( new QueryEndHttpNotifier(getCliService().getHiveConf(), this.logSegregationContext), QueryEnded.class); log.info("Registered query result formatter"); }
/** * Create an asynchronous event listener which uses a thread poool to process events. * * @param poolSize size of the event processing pool * @param timeOutSeconds time out in seconds when an idle thread is destroyed * @param isDaemon if the threads used to process should be daemon threads, * if false, then implementation should call stop() * to stop the thread pool */ public AsyncEventListener(int poolSize, long timeOutSeconds, final boolean isDaemon) { eventQueue = new LinkedBlockingQueue<>(); ThreadFactory factory = new BasicThreadFactory.Builder() .namingPattern(getName()+"_AsyncThread-%d") .daemon(isDaemon) .priority(Thread.NORM_PRIORITY) .build(); // fixed pool with min and max equal to poolSize processor = new ThreadPoolExecutor(poolSize, poolSize, timeOutSeconds, TimeUnit.SECONDS, eventQueue, factory); processor.allowCoreThreadTimeOut(true); }
getEventService().notifyEvent(event); } catch (LensException e) { log.warn("LensEventService encountered error while handling event: {}", event.getEventId(), e);
@Override public synchronized void init(HiveConf hiveConf) { super.init(hiveConf); maxJobsPerUser = hiveConf.getInt(LensConfConstants.MAX_SCHEDULED_JOB_PER_USER, maxJobsPerUser); try { schedulerDAO = new SchedulerDAO(hiveConf); waitingThreadInterval = hiveConf.getLong(LensConfConstants.SCHEDULED_INSTANCE_WAITING_THREAD_INTERVAL_MILLIS, LensConfConstants.DEFAULT_SCHEDULED_INSTANCE_WAITING_THREAD_INTERVAL_MILLIS); alarmService = LensServices.get().getService(AlarmService.NAME); queryService = LensServices.get().getService(QueryExecutionService.NAME); this.schedulerEventListener = new SchedulerEventListener(schedulerDAO); this.schedulerQueryEventListener = new SchedulerQueryEventListener(schedulerDAO); getEventService().addListenerForType(schedulerEventListener, SchedulerAlarmEvent.class); getEventService().addListenerForType(schedulerQueryEventListener, QueryEnded.class); this.waitingInstanceThread = new Thread( new WaitingInstanceHandler(getEventService(), schedulerDAO, waitingThreadInterval)); } catch (LensException e) { log.error("Error Initialising Scheduler-service", e); healthy = false; } }