private void createTimelineDomain(String domainId, String readers, String writers) throws Exception { TimelineDomain timelineDomain = new TimelineDomain(); timelineDomain.setId(domainId); timelineDomain.setReaders(readers); timelineDomain.setWriters(writers); timelineClient.putDomain(timelineDomain); LOG.info("ATS domain created:" + domainId + "(" + readers + "," + writers + ")"); }
private static void publishApplicationAttemptEvent( final TimelineClient timelineClient, String appAttemptId, DSEvent appEvent, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(appAttemptId); entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setEventType(appEvent.toString()); event.setTimestamp(System.currentTimeMillis()); entity.addEvent(event); try { timelineClient.putEntities(entity); } catch (YarnException e) { LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? JOYConstants.START : JOYConstants.END) + " event could not be published for " + appAttemptId.toString(), e); } catch (IOException e) { LOG.error("App Attempt " + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? JOYConstants.START : JOYConstants.END) + " event could not be published for " + appAttemptId.toString(), e); } }
@Test public void testCreatePreHookEventJsonShhouldMatch() throws Exception { TimelineEntity timelineEntity = uut.createPreHookEvent( "test-query-id", "test-query", new org.json.JSONObject(), 0L, "test-user", "test-request-user", 0, 0, "test-opid", "client-ip-address", "hive-instance-address", "hive-instance-type", "session-id", "log-id", "thread-id", "execution-mode", Collections.<String>emptyList(), Collections.<String>emptyList(), new HiveConf(), null, "domain-id"); String resultStr = (String) timelineEntity.getOtherInfo() .get(ATSHook.OtherInfoTypes.QUERY.name()); JsonNode result = objectMapper.readTree(resultStr); JsonNode expected = objectMapper.readTree("{\"queryText\":\"test-query\"," + "\"queryPlan\":{}}"); assertEquals(expected, result); } }
/** * Handle error and set it in response. */ private static void handleError(TimelineEntity entity, TimelinePutResponse response, final int errorCode) { TimelinePutError error = new TimelinePutError(); error.setEntityId(entity.getEntityId()); error.setEntityType(entity.getEntityType()); error.setErrorCode(errorCode); response.addError(error); }
public static TimelineDomain generateDomain() { TimelineDomain domain = new TimelineDomain(); domain.setId("namesapce id"); domain.setDescription("domain description"); domain.setOwner("domain owner"); domain.setReaders("domain_reader"); domain.setWriters("domain_writer"); domain.setCreatedTime(0L); domain.setModifiedTime(1L); return domain; }
public static TimelineEntityGroupId newInstance(ApplicationId applicationId, String id) { TimelineEntityGroupId timelineEntityGroupId = new TimelineEntityGroupId(); timelineEntityGroupId.setApplicationId(applicationId); timelineEntityGroupId.setTimelineEntityGroupId(id); return timelineEntityGroupId; }
TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, String requestuser, boolean success, String opId, Map<String, Long> durations, String domainId) throws Exception { LOG.info("Received post-hook notification for :" + queryId); TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(queryId); atsEntity.setEntityType(EntityTypes.HIVE_QUERY_ID.name()); atsEntity.addPrimaryFilter(PrimaryFilterTypes.user.name(), user); atsEntity.addPrimaryFilter(PrimaryFilterTypes.requestuser.name(), requestuser); if (opId != null) { atsEntity.addPrimaryFilter(PrimaryFilterTypes.operationid.name(), opId); } TimelineEvent stopEvt = new TimelineEvent(); stopEvt.setEventType(EventTypes.QUERY_COMPLETED.name()); stopEvt.setTimestamp(stopTime); atsEntity.addEvent(stopEvt); atsEntity.addOtherInfo(OtherInfoTypes.STATUS.name(), success); // Perf times JSONObject perfObj = new JSONObject(new LinkedHashMap<>()); for (Map.Entry<String, Long> entry : durations.entrySet()) { perfObj.put(entry.getKey(), entry.getValue()); } atsEntity.addOtherInfo(OtherInfoTypes.PERF.name(), perfObj.toString()); atsEntity.setDomainId(domainId); return atsEntity; }
private void createTimelineDomain(String domainId, String readers, String writers) throws Exception { TimelineDomain timelineDomain = new TimelineDomain(); timelineDomain.setId(domainId); timelineDomain.setReaders(readers); timelineDomain.setWriters(writers); timelineClient.putDomain(timelineDomain); LOG.info("ATS domain created:" + domainId + "(" + readers + "," + writers + ")"); }
/** * Handle error and set it in response. */ private static void handleError(TimelineEntity entity, TimelinePutResponse response, final int errorCode) { TimelinePutError error = new TimelinePutError(); error.setEntityId(entity.getEntityId()); error.setEntityType(entity.getEntityType()); error.setErrorCode(errorCode); response.addError(error); }
TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, String requestuser, boolean success, String opId, Map<String, Long> durations, String domainId) throws Exception { LOG.info("Received post-hook notification for :" + queryId); TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(queryId); atsEntity.setEntityType(EntityTypes.HIVE_QUERY_ID.name()); atsEntity.addPrimaryFilter(PrimaryFilterTypes.user.name(), user); atsEntity.addPrimaryFilter(PrimaryFilterTypes.requestuser.name(), requestuser); if (opId != null) { atsEntity.addPrimaryFilter(PrimaryFilterTypes.operationid.name(), opId); } TimelineEvent stopEvt = new TimelineEvent(); stopEvt.setEventType(EventTypes.QUERY_COMPLETED.name()); stopEvt.setTimestamp(stopTime); atsEntity.addEvent(stopEvt); atsEntity.addOtherInfo(OtherInfoTypes.STATUS.name(), success); // Perf times JSONObject perfObj = new JSONObject(new LinkedHashMap<>()); for (Map.Entry<String, Long> entry : durations.entrySet()) { perfObj.put(entry.getKey(), entry.getValue()); } atsEntity.addOtherInfo(OtherInfoTypes.PERF.name(), perfObj.toString()); atsEntity.setDomainId(domainId); return atsEntity; }
private void prepareTimelineDomain() { TimelineClient timelineClient = null; if (jstormClientContext.conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(jstormClientContext.conf); timelineClient.start(); } else { LOG.warn("Cannot put the domain " + jstormClientContext.domainId + " because the timeline service is not enabled"); return; } try { TimelineDomain domain = new TimelineDomain(); domain.setId(jstormClientContext.domainId); domain.setReaders( jstormClientContext.viewACLs != null && jstormClientContext.viewACLs.length() > 0 ? jstormClientContext.viewACLs : JOYConstants.BLANK); domain.setWriters( jstormClientContext.modifyACLs != null && jstormClientContext.modifyACLs.length() > 0 ? jstormClientContext.modifyACLs : JOYConstants.BLANK); timelineClient.putDomain(domain); LOG.info("Put the timeline domain: " + TimelineUtils.dumpTimelineRecordtoJSON(domain)); } catch (Exception e) { LOG.error("Error when putting the timeline domain", e); } finally { timelineClient.stop(); } }
/** * Handle error and set it in response. */ private static void handleError(TimelineEntity entity, TimelinePutResponse response, final int errorCode) { TimelinePutError error = new TimelinePutError(); error.setEntityId(entity.getEntityId()); error.setEntityType(entity.getEntityType()); error.setErrorCode(errorCode); response.addError(error); }
private static void publishContainerEndEvent( final TimelineClient timelineClient, ContainerStatus container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getContainerId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_END.toString()); event.addEventInfo(JOYConstants.STATE, container.getState().name()); event.addEventInfo(JOYConstants.EXIT_STATE, container.getExitStatus()); entity.addEvent(event); try { timelineClient.putEntities(entity); } catch (YarnException e) { LOG.error("Container end event could not be published for " + container.getContainerId().toString(), e); } catch (IOException e) { LOG.error("Container end event could not be published for " + container.getContainerId().toString(), e); } }
private void prepareTimelineDomain() { TimelineClient timelineClient = null; if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) { timelineClient = TimelineClient.createTimelineClient(); timelineClient.init(conf); timelineClient.start(); } else { LOG.warn("Cannot put the domain " + domainId + " because the timeline service is not enabled"); return; } try { TimelineDomain domain = new TimelineDomain(); domain.setId(domainId); domain.setReaders( viewACLs != null && viewACLs.length() > 0 ? viewACLs : " "); domain.setWriters( modifyACLs != null && modifyACLs.length() > 0 ? modifyACLs : " "); timelineClient.putDomain(domain); LOG.info("Put the timeline domain: " + TimelineUtils.dumpTimelineRecordtoJSON(domain)); } catch (Exception e) { LOG.error("Error when putting the timeline domain", e); } finally { timelineClient.stop(); } } }
private static void publishContainerStartEvent( final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getId().toString()); entity.setEntityType(DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter(JOYConstants.USER, ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(DSEvent.DS_CONTAINER_START.toString()); event.addEventInfo(JOYConstants.NODE, container.getNodeId().toString()); event.addEventInfo(JOYConstants.RESOURCES, container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } }
TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(queryId); atsEntity.setEntityType(EntityTypes.HIVE_QUERY_ID.name()); atsEntity.addPrimaryFilter(PrimaryFilterTypes.user.name(), user); atsEntity.addPrimaryFilter(PrimaryFilterTypes.requestuser.name(), requestuser); atsEntity.addPrimaryFilter(PrimaryFilterTypes.executionmode.name(), executionMode); atsEntity.addPrimaryFilter(PrimaryFilterTypes.queue.name(), conf.get("mapreduce.job.queuename")); atsEntity.addPrimaryFilter(PrimaryFilterTypes.operationid.name(), opId); atsEntity.addPrimaryFilter(PrimaryFilterTypes.tablesread.name(), tabName); atsEntity.addPrimaryFilter(PrimaryFilterTypes.tableswritten.name(), tabName); TimelineEvent startEvt = new TimelineEvent(); startEvt.setEventType(EventTypes.QUERY_SUBMITTED.name()); startEvt.setTimestamp(startTime); atsEntity.addEvent(startEvt); atsEntity.addOtherInfo(OtherInfoTypes.QUERY.name(), queryObj.toString()); atsEntity.addOtherInfo(OtherInfoTypes.TEZ.name(), numTezJobs > 0); atsEntity.addOtherInfo(OtherInfoTypes.MAPRED.name(), numMrJobs > 0); atsEntity.addOtherInfo(OtherInfoTypes.SESSION_ID.name(), sessionID); atsEntity.addOtherInfo(OtherInfoTypes.INVOKER_INFO.name(), logID); atsEntity.addOtherInfo(OtherInfoTypes.THREAD_NAME.name(), threadId); atsEntity.addOtherInfo(OtherInfoTypes.VERSION.name(), VERSION); if (clientIpAddress != null) { atsEntity.addOtherInfo(OtherInfoTypes.CLIENT_IP_ADDRESS.name(), clientIpAddress);
TimelineEntity atsEntity = new TimelineEntity(); atsEntity.setEntityId(queryId); atsEntity.setEntityType(EntityTypes.HIVE_QUERY_ID.name()); atsEntity.addPrimaryFilter(PrimaryFilterTypes.user.name(), user); atsEntity.addPrimaryFilter(PrimaryFilterTypes.requestuser.name(), requestuser); atsEntity.addPrimaryFilter(PrimaryFilterTypes.executionmode.name(), executionMode); atsEntity.addPrimaryFilter(PrimaryFilterTypes.queue.name(), conf.get("mapreduce.job.queuename")); atsEntity.addPrimaryFilter(PrimaryFilterTypes.operationid.name(), opId); atsEntity.addPrimaryFilter(PrimaryFilterTypes.tablesread.name(), tabName); atsEntity.addPrimaryFilter(PrimaryFilterTypes.tableswritten.name(), tabName); TimelineEvent startEvt = new TimelineEvent(); startEvt.setEventType(EventTypes.QUERY_SUBMITTED.name()); startEvt.setTimestamp(startTime); atsEntity.addEvent(startEvt); atsEntity.addOtherInfo(OtherInfoTypes.QUERY.name(), queryObj.toString()); atsEntity.addOtherInfo(OtherInfoTypes.TEZ.name(), numTezJobs > 0); atsEntity.addOtherInfo(OtherInfoTypes.MAPRED.name(), numMrJobs > 0); atsEntity.addOtherInfo(OtherInfoTypes.SESSION_ID.name(), sessionID); atsEntity.addOtherInfo(OtherInfoTypes.INVOKER_INFO.name(), logID); atsEntity.addOtherInfo(OtherInfoTypes.THREAD_NAME.name(), threadId); atsEntity.addOtherInfo(OtherInfoTypes.VERSION.name(), VERSION); if (clientIpAddress != null) { atsEntity.addOtherInfo(OtherInfoTypes.CLIENT_IP_ADDRESS.name(), clientIpAddress);
public void publishApplicationAttemptEvent( final TimelineClient timelineClient, String appAttemptId, ContainerEvents appEvent, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(appAttemptId); entity.setEntityType(ApplicationMaster.DSEntity.DS_APP_ATTEMPT.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter("user", ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setEventType(appEvent.toString()); event.setTimestamp(System.currentTimeMillis()); entity.addEvent(event); try { timelineClient.putEntities(entity); } catch (YarnException | IOException e) { LOG.error("App Attempt " + (appEvent.equals(ContainerEvents.APP_ATTEMPT_START) ? "start" : "end") + " event could not be published for " + appAttemptId.toString(), e); } } public void publishContainerStartEvent(
public void publishContainerEndEvent( final TimelineClient timelineClient, ContainerStatus container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId(container.getContainerId().toString()); entity.setEntityType(ApplicationMaster.DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter("user", ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(ContainerEvents.CONTAINER_END.toString()); event.addEventInfo("State", container.getState().name()); event.addEventInfo("Exit Status", container.getExitStatus()); entity.addEvent(event); try { timelineClient.putEntities(entity); } catch (YarnException | IOException e) { LOG.error("Container end event could not be published for " + container.getContainerId().toString(), e); } } public void publishApplicationAttemptEvent(
public void publishContainerStartEvent( final TimelineClient timelineClient, Container container, String domainId, UserGroupInformation ugi) { final TimelineEntity entity = new TimelineEntity(); entity.setEntityId("" + container.getId()); entity.setEntityType(ApplicationMaster.DSEntity.DS_CONTAINER.toString()); entity.setDomainId(domainId); entity.addPrimaryFilter("user", ugi.getShortUserName()); TimelineEvent event = new TimelineEvent(); event.setTimestamp(System.currentTimeMillis()); event.setEventType(ContainerEvents.CONTAINER_START.toString()); event.addEventInfo("Node", container.getNodeId().toString()); event.addEventInfo("Resources", container.getResource().toString()); entity.addEvent(event); try { ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() { @Override public TimelinePutResponse run() throws Exception { return timelineClient.putEntities(entity); } }); } catch (Exception e) { LOG.error("Container start event could not be published for " + container.getId().toString(), e instanceof UndeclaredThrowableException ? e.getCause() : e); } } }