@Override public RemotingCommand processRequest(Channel channel, RemotingCommand request) throws RemotingCommandException { JobCompletedRequest requestBody = request.getBody(); for (JobCompletedBiz biz : bizChain) { RemotingCommand remotingCommand = biz.doBiz(requestBody); if (remotingCommand != null) { return remotingCommand; } } return RemotingCommand.createResponseCommand(RemotingProtos.ResponseCode.SUCCESS.code()); }
@Override public RemotingCommand processRequest(Channel channel, RemotingCommand request) throws RemotingCommandException { JobCompletedRequest requestBody = request.getBody(); for (JobCompletedBiz biz : bizChain) { RemotingCommand remotingCommand = biz.doBiz(requestBody); if (remotingCommand != null) { return remotingCommand; } } return RemotingCommand.createResponseCommand(RemotingProtos.ResponseCode.SUCCESS.code()); }
/** * 1. 将 channel 纳入管理中(不存在就加入) * 2. 更新 TaskTracker 节点信息(可用线程数) */ private void offerHandler(Channel channel, RemotingCommand request) { AbstractRemotingCommandBody commandBody = request.getBody(); String nodeGroup = commandBody.getNodeGroup(); String identity = commandBody.getIdentity(); NodeType nodeType = NodeType.valueOf(commandBody.getNodeType()); // 1. 将 channel 纳入管理中(不存在就加入) appContext.getChannelManager().offerChannel(new ChannelWrapper(channel, nodeType, nodeGroup, identity)); }
/** * 1. 将 channel 纳入管理中(不存在就加入) * 2. 更新 TaskTracker 节点信息(可用线程数) */ private void offerHandler(Channel channel, RemotingCommand request) { AbstractRemotingCommandBody commandBody = request.getBody(); String nodeGroup = commandBody.getNodeGroup(); String identity = commandBody.getIdentity(); NodeType nodeType = NodeType.valueOf(commandBody.getNodeType()); // 1. 将 channel 纳入管理中(不存在就加入) appContext.getChannelManager().offerChannel(new ChannelWrapper(channel, nodeType, nodeGroup, identity)); }
@Override public RemotingCommand processRequest(Channel Channel, RemotingCommand request) throws RemotingCommandException { JobFinishedRequest requestBody = request.getBody(); try { if (appContext.getJobCompletedHandler() != null) { appContext.getJobCompletedHandler().onComplete(requestBody.getJobResults()); stat.incHandleFeedbackNum(CollectionUtils.sizeOf(requestBody.getJobResults())); } } catch (Exception t) { LOGGER.error(t.getMessage(), t); } return RemotingCommand.createResponseCommand(JobProtos.ResponseCode.JOB_NOTIFY_SUCCESS.code(), "received successful"); } }
@Override public RemotingCommand processRequest(Channel Channel, RemotingCommand request) throws RemotingCommandException { JobFinishedRequest requestBody = request.getBody(); try { if (appContext.getJobCompletedHandler() != null) { appContext.getJobCompletedHandler().onComplete(requestBody.getJobResults()); stat.incHandleFeedbackNum(CollectionUtils.sizeOf(requestBody.getJobResults())); } } catch (Exception t) { LOGGER.error(t.getMessage(), t); } return RemotingCommand.createResponseCommand(JobProtos.ResponseCode.JOB_NOTIFY_SUCCESS.code(), "received successful"); } }
@Override public void operationComplete(ResponseFuture responseFuture) { RemotingCommand response = responseFuture.getResponseCommand(); if (response != null && RemotingProtos.ResponseCode.SUCCESS.code() == response.getCode()) { JobAskResponse responseBody = response.getBody(); List<String> deadJobIds = responseBody.getJobIds(); if (CollectionUtils.isNotEmpty(deadJobIds)) { // 睡了1秒再修复, 防止任务刚好执行完正在传输中. 1s可以让完成的正常完成 QuietUtils.sleep(appContext.getConfig().getParameter(ExtConfig.JOB_TRACKER_FIX_EXECUTING_JOB_WAITING_MILLS, 1000L)); for (JobPo jobPo : jobPos) { if (deadJobIds.contains(jobPo.getJobId())) { fixDeadJob(jobPo); } } } } } });
@Override public RemotingCommand processRequest(final Channel ctx, final RemotingCommand request) throws RemotingCommandException { JobPullRequest requestBody = request.getBody(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("taskTrackerNodeGroup:{}, taskTrackerIdentity:{} , availableThreads:{}", requestBody.getNodeGroup(), requestBody.getIdentity(), requestBody.getAvailableThreads()); } jobPusher.push(requestBody); return RemotingCommand.createResponseCommand(JobProtos.ResponseCode.JOB_PULL_SUCCESS.code(), ""); } }
@Override public RemotingCommand processRequest(final Channel ctx, final RemotingCommand request) throws RemotingCommandException { JobPullRequest requestBody = request.getBody(); if (LOGGER.isDebugEnabled()) { LOGGER.debug("taskTrackerNodeGroup:{}, taskTrackerIdentity:{} , availableThreads:{}", requestBody.getNodeGroup(), requestBody.getIdentity(), requestBody.getAvailableThreads()); } jobPusher.push(requestBody); return RemotingCommand.createResponseCommand(JobProtos.ResponseCode.JOB_PULL_SUCCESS.code(), ""); } }
@Override public void operationComplete(ResponseFuture responseFuture) { RemotingCommand response = responseFuture.getResponseCommand(); if (response != null && RemotingProtos.ResponseCode.SUCCESS.code() == response.getCode()) { JobAskResponse responseBody = response.getBody(); List<String> deadJobIds = responseBody.getJobIds(); if (CollectionUtils.isNotEmpty(deadJobIds)) { // 睡了1秒再修复, 防止任务刚好执行完正在传输中. 1s可以让完成的正常完成 QuietUtils.sleep(appContext.getConfig().getParameter(ExtConfig.JOB_TRACKER_FIX_EXECUTING_JOB_WAITING_MILLS, 1000L)); for (JobPo jobPo : jobPos) { if (deadJobIds.contains(jobPo.getJobId())) { fixDeadJob(jobPo); } } } } } });
@Override public RemotingCommand processRequest(Channel channel, final RemotingCommand request) throws RemotingCommandException { JobPushRequest requestBody = request.getBody(); // JobTracker 分发来的 job final List<JobMeta> jobMetaList = requestBody.getJobMetaList(); List<String> failedJobIds = null; for (JobMeta jobMeta : jobMetaList) { try { appContext.getRunnerPool().execute(jobMeta, jobRunnerCallback); } catch (NoAvailableJobRunnerException e) { if (failedJobIds == null) { failedJobIds = new ArrayList<String>(); } failedJobIds.add(jobMeta.getJobId()); } } if (CollectionUtils.isNotEmpty(failedJobIds)) { // 任务推送失败 JobPushResponse jobPushResponse = new JobPushResponse(); jobPushResponse.setFailedJobIds(failedJobIds); return RemotingCommand.createResponseCommand(JobProtos.ResponseCode.NO_AVAILABLE_JOB_RUNNER.code(), jobPushResponse); } // 任务推送成功 return RemotingCommand.createResponseCommand(JobProtos .ResponseCode.JOB_PUSH_SUCCESS.code(), "job push success!"); }
@Override public RemotingCommand processRequest(Channel channel, final RemotingCommand request) throws RemotingCommandException { JobPushRequest requestBody = request.getBody(); // JobTracker 分发来的 job final List<JobMeta> jobMetaList = requestBody.getJobMetaList(); List<String> failedJobIds = null; for (JobMeta jobMeta : jobMetaList) { try { appContext.getRunnerPool().execute(jobMeta, jobRunnerCallback); } catch (NoAvailableJobRunnerException e) { if (failedJobIds == null) { failedJobIds = new ArrayList<String>(); } failedJobIds.add(jobMeta.getJobId()); } } if (CollectionUtils.isNotEmpty(failedJobIds)) { // 任务推送失败 JobPushResponse jobPushResponse = new JobPushResponse(); jobPushResponse.setFailedJobIds(failedJobIds); return RemotingCommand.createResponseCommand(JobProtos.ResponseCode.NO_AVAILABLE_JOB_RUNNER.code(), jobPushResponse); } // 任务推送成功 return RemotingCommand.createResponseCommand(JobProtos .ResponseCode.JOB_PUSH_SUCCESS.code(), "job push success!"); }
@Override public RemotingCommand processRequest(Channel channel, RemotingCommand request) throws RemotingCommandException { JobAskRequest requestBody = request.getBody(); List<String> jobIds = requestBody.getJobIds(); List<String> notExistJobIds = appContext.getRunnerPool() .getRunningJobManager().getNotExists(jobIds); JobAskResponse responseBody = CommandBodyWrapper.wrapper(appContext, new JobAskResponse()); responseBody.setJobIds(notExistJobIds); return RemotingCommand.createResponseCommand( RemotingProtos.ResponseCode.SUCCESS.code(), "查询成功", responseBody); } }
@Override public RemotingCommand processRequest(Channel channel, RemotingCommand request) throws RemotingCommandException { JobAskRequest requestBody = request.getBody(); List<String> jobIds = requestBody.getJobIds(); List<String> notExistJobIds = appContext.getRunnerPool() .getRunningJobManager().getNotExists(jobIds); JobAskResponse responseBody = CommandBodyWrapper.wrapper(appContext, new JobAskResponse()); responseBody.setJobIds(notExistJobIds); return RemotingCommand.createResponseCommand( RemotingProtos.ResponseCode.SUCCESS.code(), "查询成功", responseBody); } }
@Override public RemotingCommand processRequest(Channel channel, RemotingCommand request) throws RemotingCommandException { JobSubmitRequest jobSubmitRequest = request.getBody(); JobSubmitResponse jobSubmitResponse = appContext.getCommandBodyWrapper().wrapper(new JobSubmitResponse()); RemotingCommand response; try { appContext.getJobReceiver().receive(jobSubmitRequest); response = RemotingCommand.createResponseCommand( JobProtos.ResponseCode.JOB_RECEIVE_SUCCESS.code(), "job submit success!", jobSubmitResponse); } catch (JobReceiveException e) { LOGGER.error("Receive job failed , jobs = " + jobSubmitRequest.getJobs(), e); jobSubmitResponse.setSuccess(false); jobSubmitResponse.setMsg(e.getMessage()); jobSubmitResponse.setFailedJobs(e.getJobs()); response = RemotingCommand.createResponseCommand( JobProtos.ResponseCode.JOB_RECEIVE_FAILED.code(), e.getMessage(), jobSubmitResponse); } return response; } }
@Override public RemotingCommand processRequest(Channel channel, RemotingCommand request) throws RemotingCommandException { JobSubmitRequest jobSubmitRequest = request.getBody(); JobSubmitResponse jobSubmitResponse = appContext.getCommandBodyWrapper().wrapper(new JobSubmitResponse()); RemotingCommand response; try { appContext.getJobReceiver().receive(jobSubmitRequest); response = RemotingCommand.createResponseCommand( JobProtos.ResponseCode.JOB_RECEIVE_SUCCESS.code(), "job submit success!", jobSubmitResponse); } catch (JobReceiveException e) { LOGGER.error("Receive job failed , jobs = " + jobSubmitRequest.getJobs(), e); jobSubmitResponse.setSuccess(false); jobSubmitResponse.setMsg(e.getMessage()); jobSubmitResponse.setFailedJobs(e.getJobs()); response = RemotingCommand.createResponseCommand( JobProtos.ResponseCode.JOB_RECEIVE_FAILED.code(), e.getMessage(), jobSubmitResponse); } return response; } }
@Override public void call(RemotingCommand responseCommand) { if (responseCommand == null) { response.setFailedJobs(jobs); response.setSuccess(false); response.setMsg("Submit Job failed: JobTracker is broken"); LOGGER.warn("Submit Job failed: {}, {}", jobs, "JobTracker is broken"); return; } if (JobProtos.ResponseCode.JOB_RECEIVE_SUCCESS.code() == responseCommand.getCode()) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Submit Job success: {}", jobs); } response.setSuccess(true); return; } // 失败的job JobSubmitResponse jobSubmitResponse = responseCommand.getBody(); response.setFailedJobs(jobSubmitResponse.getFailedJobs()); response.setSuccess(false); response.setCode(JobProtos.ResponseCode.valueOf(responseCommand.getCode()).name()); response.setMsg("Submit Job failed: " + responseCommand.getRemark() + " " + jobSubmitResponse.getMsg()); LOGGER.warn("Submit Job failed: {}, {}, {}", jobs, responseCommand.getRemark(), jobSubmitResponse.getMsg()); } };
@Override public void call(RemotingCommand responseCommand) { if (responseCommand == null) { response.setFailedJobs(jobs); response.setSuccess(false); response.setMsg("Submit Job failed: JobTracker is broken"); LOGGER.warn("Submit Job failed: {}, {}", jobs, "JobTracker is broken"); return; } if (JobProtos.ResponseCode.JOB_RECEIVE_SUCCESS.code() == responseCommand.getCode()) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Submit Job success: {}", jobs); } response.setSuccess(true); return; } // 失败的job JobSubmitResponse jobSubmitResponse = responseCommand.getBody(); response.setFailedJobs(jobSubmitResponse.getFailedJobs()); response.setSuccess(false); response.setCode(JobProtos.ResponseCode.valueOf(responseCommand.getCode()).name()); response.setMsg("Submit Job failed: " + responseCommand.getRemark() + " " + jobSubmitResponse.getMsg()); LOGGER.warn("Submit Job failed: {}, {}, {}", jobs, responseCommand.getRemark(), jobSubmitResponse.getMsg()); } };
@Override public RemotingCommand processRequest(Channel channel, RemotingCommand request) throws RemotingCommandException { BizLogSendRequest requestBody = request.getBody(); List<BizLog> bizLogs = requestBody.getBizLogs(); if (CollectionUtils.isNotEmpty(bizLogs)) { for (BizLog bizLog : bizLogs) { JobLogPo jobLogPo = new JobLogPo(); jobLogPo.setGmtCreated(SystemClock.now()); jobLogPo.setLogTime(bizLog.getLogTime()); jobLogPo.setTaskTrackerNodeGroup(bizLog.getTaskTrackerNodeGroup()); jobLogPo.setTaskTrackerIdentity(bizLog.getTaskTrackerIdentity()); jobLogPo.setJobId(bizLog.getJobId()); jobLogPo.setTaskId(bizLog.getTaskId()); jobLogPo.setRealTaskId(bizLog.getRealTaskId()); jobLogPo.setJobType(bizLog.getJobType()); jobLogPo.setMsg(bizLog.getMsg()); jobLogPo.setSuccess(true); jobLogPo.setLevel(bizLog.getLevel()); jobLogPo.setLogType(LogType.BIZ); appContext.getJobLogger().log(jobLogPo); } } return RemotingCommand.createResponseCommand(JobProtos.ResponseCode.BIZ_LOG_SEND_SUCCESS.code(), ""); } }
@Override public RemotingCommand processRequest(Channel channel, RemotingCommand request) throws RemotingCommandException { BizLogSendRequest requestBody = request.getBody(); List<BizLog> bizLogs = requestBody.getBizLogs(); if (CollectionUtils.isNotEmpty(bizLogs)) { for (BizLog bizLog : bizLogs) { JobLogPo jobLogPo = new JobLogPo(); jobLogPo.setGmtCreated(SystemClock.now()); jobLogPo.setLogTime(bizLog.getLogTime()); jobLogPo.setTaskTrackerNodeGroup(bizLog.getTaskTrackerNodeGroup()); jobLogPo.setTaskTrackerIdentity(bizLog.getTaskTrackerIdentity()); jobLogPo.setJobId(bizLog.getJobId()); jobLogPo.setTaskId(bizLog.getTaskId()); jobLogPo.setRealTaskId(bizLog.getRealTaskId()); jobLogPo.setJobType(bizLog.getJobType()); jobLogPo.setMsg(bizLog.getMsg()); jobLogPo.setSuccess(true); jobLogPo.setLevel(bizLog.getLevel()); jobLogPo.setLogType(LogType.BIZ); appContext.getJobLogger().log(jobLogPo); } } return RemotingCommand.createResponseCommand(JobProtos.ResponseCode.BIZ_LOG_SEND_SUCCESS.code(), ""); } }