FileSystem fs = path.getFileSystem(conf); AtomicLong time = new AtomicLong(); EventLogger evtLogger = new EventLogger(conf, () -> time.get()); evtLogger.handle(context); int statusLen = 0;
List<ExecDriver> mrTasks = Utilities.getMRTasks(plan.getRootTasks()); List<TezTask> tezTasks = Utilities.getTezTasks(plan.getRootTasks()); ExecutionMode executionMode = getExecutionMode(plan, mrTasks, tezTasks); builder.setTimestamp(plan.getQueryStartTime()); builder.setHiveQueryId(plan.getQueryId()); builder.setUser(getUser(hookContext)); builder.setRequestUser(getRequestUser(hookContext)); String queueName = getQueueName(executionMode, conf); if (queueName != null) { builder.setQueue(queueName); builder.addAllTablesRead(getTablesFromEntitySet(hookContext.getInputs())); builder.addAllTablesWritten(getTablesFromEntitySet(hookContext.getOutputs())); if (hookContext.getOperationId() != null) { builder.setOperationId(hookContext.getOperationId()); JSONObject queryObj = new JSONObject(); queryObj.put("queryText", plan.getQueryStr()); queryObj.put("queryPlan", getExplainPlan(plan, conf, hookContext)); addMapEntry(builder, OtherInfoType.QUERY, queryObj.toString()); } catch (Exception e) { LOG.error("Unexpected exception while serializing json.", e); addMapEntry(builder, OtherInfoType.TEZ, Boolean.toString(tezTasks.size() > 0)); addMapEntry(builder, OtherInfoType.MAPRED, Boolean.toString(mrTasks.size() > 0)); addMapEntry(builder, OtherInfoType.SESSION_ID, hookContext.getSessionId()); String logID = conf.getLogIdVar(hookContext.getSessionId()); addMapEntry(builder, OtherInfoType.INVOKER_INFO, logID);
@Override public void run(HookContext hookContext) throws Exception { try { EventLogger logger = EventLogger.getInstance(hookContext.getConf()); logger.handle(hookContext); } catch (Exception e) { LOG.error("Got exceptoin while processing event: ", e); } } }
switch (hookContext.getHookType()) { case PRE_EXEC_HOOK: event = getPreHookEvent(hookContext); break; case POST_EXEC_HOOK: event = getPostHookEvent(hookContext, true); break; case ON_FAILURE_HOOK: event = getPostHookEvent(hookContext, false); break; default: logWriter.execute(() -> writeEvent(event)); } catch (RejectedExecutionException e) { LOG.warn("Writer queue full ignoring event {} for query {}",
@Test public void testQueueLogs() throws Exception { context.setHookType(HookType.PRE_EXEC_HOOK); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); context.getQueryPlan().getRootTasks().add(task); context.getQueryPlan().getRootTasks().add(new TezTask()); evtLogger.handle(context); evtLogger.handle(context); evtLogger.shutdown();
@Test public void testPreEventLog() throws Exception { context.setHookType(HookType.PRE_EXEC_HOOK); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); evtLogger.shutdown(); HiveHookEventProto event = loadEvent(conf, tmpFolder); Assert.assertEquals(EventType.QUERY_SUBMITTED.name(), event.getEventType()); Assert.assertEquals(1234L, event.getTimestamp()); Assert.assertEquals(System.getProperty("user.name"), event.getUser()); Assert.assertEquals("test_user", event.getRequestUser()); Assert.assertEquals("test_queryId", event.getHiveQueryId()); Assert.assertEquals("test_op_id", event.getOperationId()); Assert.assertEquals(ExecutionMode.NONE.name(), event.getExecutionMode()); Assert.assertFalse(event.hasQueue()); assertOtherInfo(event, OtherInfoType.TEZ, Boolean.FALSE.toString()); assertOtherInfo(event, OtherInfoType.MAPRED, Boolean.FALSE.toString()); assertOtherInfo(event, OtherInfoType.CLIENT_IP_ADDRESS, "192.168.10.10"); assertOtherInfo(event, OtherInfoType.SESSION_ID, "test_session_id"); assertOtherInfo(event, OtherInfoType.THREAD_NAME, "test_thread_id"); assertOtherInfo(event, OtherInfoType.HIVE_INSTANCE_TYPE, "HS2"); assertOtherInfo(event, OtherInfoType.HIVE_ADDRESS, "hive_addr"); assertOtherInfo(event, OtherInfoType.CONF, null); assertOtherInfo(event, OtherInfoType.QUERY, null); }
@Test public void testPostEventLog() throws Exception { context.setHookType(HookType.POST_EXEC_HOOK); context.getPerfLogger().PerfLogBegin("test", "LogTest"); context.getPerfLogger().PerfLogEnd("test", "LogTest"); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); evtLogger.shutdown(); HiveHookEventProto event = loadEvent(conf, tmpFolder); Assert.assertEquals(EventType.QUERY_COMPLETED.name(), event.getEventType()); Assert.assertEquals(System.getProperty("user.name"), event.getUser()); Assert.assertEquals("test_user", event.getRequestUser()); Assert.assertEquals("test_queryId", event.getHiveQueryId()); Assert.assertEquals("test_op_id", event.getOperationId()); assertOtherInfo(event, OtherInfoType.STATUS, Boolean.TRUE.toString()); String val = findOtherInfo(event, OtherInfoType.PERF); Map<String, Long> map = new ObjectMapper().readValue(val, new TypeReference<Map<String, Long>>() {}); // This should be really close to zero. Assert.assertTrue("Expected LogTest in PERF", map.get("LogTest") < 100); }
private HiveHookEventProto getPostHookEvent(HookContext hookContext, boolean success) { QueryPlan plan = hookContext.getQueryPlan(); LOG.info("Received post-hook notification for: " + plan.getQueryId()); HiveHookEventProto.Builder builder = HiveHookEventProto.newBuilder(); builder.setEventType(EventType.QUERY_COMPLETED.name()); builder.setTimestamp(clock.getTime()); builder.setHiveQueryId(plan.getQueryId()); builder.setUser(getUser(hookContext)); builder.setRequestUser(getRequestUser(hookContext)); if (hookContext.getOperationId() != null) { builder.setOperationId(hookContext.getOperationId()); } addMapEntry(builder, OtherInfoType.STATUS, Boolean.toString(success)); JSONObject perfObj = new JSONObject(); for (String key : hookContext.getPerfLogger().getEndTimes().keySet()) { perfObj.put(key, hookContext.getPerfLogger().getDuration(key)); } addMapEntry(builder, OtherInfoType.PERF, perfObj.toString()); return builder.build(); }
@Test public void testPreAndPostEventBoth() throws Exception { context.setHookType(HookType.PRE_EXEC_HOOK); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); context.setHookType(HookType.POST_EXEC_HOOK); evtLogger.handle(context); evtLogger.shutdown(); ProtoMessageReader<HiveHookEventProto> reader = getTestReader(conf, tmpFolder); HiveHookEventProto event = reader.readEvent(); Assert.assertNotNull("Pre hook event not found", event); Assert.assertEquals(EventType.QUERY_SUBMITTED.name(), event.getEventType()); event = reader.readEvent(); Assert.assertNotNull("Post hook event not found", event); Assert.assertEquals(EventType.QUERY_COMPLETED.name(), event.getEventType()); }
@Test public void testFailureEventLog() throws Exception { context.setHookType(HookType.ON_FAILURE_HOOK); EventLogger evtLogger = new EventLogger(conf, SystemClock.getInstance()); evtLogger.handle(context); evtLogger.shutdown(); HiveHookEventProto event = loadEvent(conf, tmpFolder); Assert.assertEquals(EventType.QUERY_COMPLETED.name(), event.getEventType()); Assert.assertEquals(System.getProperty("user.name"), event.getUser()); Assert.assertEquals("test_user", event.getRequestUser()); Assert.assertEquals("test_queryId", event.getHiveQueryId()); Assert.assertEquals("test_op_id", event.getOperationId()); assertOtherInfo(event, OtherInfoType.STATUS, Boolean.FALSE.toString()); assertOtherInfo(event, OtherInfoType.PERF, null); }
static EventLogger getInstance(HiveConf conf) { if (instance == null) { synchronized (EventLogger.class) { if (instance == null) { instance = new EventLogger(conf, SystemClock.getInstance()); ShutdownHookManager.addShutdownHook(instance::shutdown); } } } return instance; } }