public Query getQuery(QueryId queryId) { return queryMasterTasks.get(queryId).getQuery(); }
private TajoHeartbeatRequest buildTajoHeartBeat(QueryMasterTask queryMasterTask) { TajoHeartbeatRequest.Builder builder = TajoHeartbeatRequest.newBuilder(); builder.setConnectionInfo(workerContext.getConnectionInfo().getProto()); builder.setQueryId(queryMasterTask.getQueryId().getProto()); builder.setState(queryMasterTask.getState()); if (queryMasterTask.getQuery() != null) { if (queryMasterTask.getQuery().getResultDesc() != null) { builder.setResultDesc(queryMasterTask.getQuery().getResultDesc().getProto()); } builder.setQueryProgress(queryMasterTask.getQuery().getProgress()); if(queryMasterTask.getQuery().getFailureReason() != null) { builder.setError(queryMasterTask.getQuery().getFailureReason()); } } if (queryMasterTask.isInitError()) { builder.setError(ErrorUtil.convertException(queryMasterTask.getInitError())); } return builder.build(); }
barrier.await(5000, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { fail("Query state : " + queryMasterTask.getQuery().getSynchronizedState()); Stage stage = queryMasterTask.getQuery().getStages().iterator().next(); assertNotNull(stage); cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_KILLED, 50); assertEquals(TajoProtos.QueryState.QUERY_KILLED, queryMasterTask.getQuery().getSynchronizedState()); } catch (Exception e) { e.printStackTrace();
public Query getQuery(QueryId queryId) { return queryMasterTasks.get(queryId).getQuery(); }
private TajoHeartbeatRequest buildTajoHeartBeat(QueryMasterTask queryMasterTask) { TajoHeartbeatRequest.Builder builder = TajoHeartbeatRequest.newBuilder(); builder.setConnectionInfo(workerContext.getConnectionInfo().getProto()); builder.setQueryId(queryMasterTask.getQueryId().getProto()); builder.setState(queryMasterTask.getState()); if (queryMasterTask.getQuery() != null) { if (queryMasterTask.getQuery().getResultDesc() != null) { builder.setResultDesc(queryMasterTask.getQuery().getResultDesc().getProto()); } builder.setQueryProgress(queryMasterTask.getQuery().getProgress()); if(queryMasterTask.getQuery().getFailureReason() != null) { builder.setError(queryMasterTask.getQuery().getFailureReason()); } } if (queryMasterTask.isInitError()) { builder.setError(ErrorUtil.convertException(queryMasterTask.getInitError())); } return builder.build(); }
barrier.await(5000, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { fail("Query state : " + queryMasterTask.getQuery().getSynchronizedState()); Stage stage = queryMasterTask.getQuery().getStages().iterator().next(); assertNotNull(stage); cluster.waitForQueryState(queryMasterTask.getQuery(), TajoProtos.QueryState.QUERY_KILLED, 50); assertEquals(TajoProtos.QueryState.QUERY_KILLED, queryMasterTask.getQuery().getSynchronizedState()); } finally { queryMasterTask.stop(); List<Stage> stages = Lists.newArrayList(queryMasterTask.getQuery().getStages()); Stage lastStage = stages.get(stages.size() - 1);
private void cleanupQuery(final QueryId queryId) { if (getQuery() != null) { Set<InetSocketAddress> workers = Sets.newHashSet(); for (Stage stage : getQuery().getStages()) { workers.addAll(stage.getAssignedWorkerMap().values()); } LOG.info("Cleanup resources of all workers. Query: " + queryId + ", workers: " + workers.size()); for (final InetSocketAddress worker : workers) { queryMasterContext.getEventExecutor().submit(new Runnable() { @Override public void run() { try { AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub(); tajoWorkerProtocolService.stopQuery(null, queryId.getProto(), NullCallback.get()); } catch (Throwable e) { LOG.error(e.getMessage(), e); } } }); } } }
private void cleanupQuery(final QueryId queryId) { if (getQuery() != null) { Set<InetSocketAddress> workers = Sets.newHashSet(); for (Stage stage : getQuery().getStages()) { workers.addAll(stage.getAssignedWorkerMap().values()); } LOG.info("Cleanup resources of all workers. Query: " + queryId + ", workers: " + workers.size()); for (final InetSocketAddress worker : workers) { queryMasterContext.getEventExecutor().submit(new Runnable() { @Override public void run() { try { AsyncRpcClient rpc = RpcClientManager.getInstance().getClient(worker, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerProtocolService = rpc.getStub(); tajoWorkerProtocolService.stopQuery(null, queryId.getProto(), NullCallback.get()); } catch (Throwable e) { LOG.error(e.getMessage(), e); } } }); } } }
public QueryHistory getQueryHistory(QueryId queryId) throws IOException { QueryMasterTask queryMasterTask = getQueryMasterTask(queryId, true); if(queryMasterTask != null) { return queryMasterTask.getQuery().getQueryHistory(); } else { return workerContext.getHistoryReader().getQueryHistory(queryId.toString()); } }
public QueryHistory getQueryHistory(QueryId queryId) throws IOException { QueryMasterTask queryMasterTask = getQueryMasterTask(queryId, true); if(queryMasterTask != null) { return queryMasterTask.getQuery().getQueryHistory(); } else { return workerContext.getHistoryReader().getQueryHistory(queryId.toString()); } }
LOG.error(e.getMessage(), e); Query query = queryMasterTask.getQuery(); if (query != null) { QueryHistory queryHisory = query.getQueryHistory();
LOG.error(e.getMessage(), e); Query query = queryMasterTask.getQuery(); if (query != null) { QueryHistory queryHisory = query.getQueryHistory();
/** * It sends a kill RPC request to a corresponding worker. * * @param workerId worker unique Id. * @param taskAttemptId The TaskAttemptId to be killed. */ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { NettyClientBase tajoWorkerRpc; ExecutionBlockId ebId = taskAttemptId.getTaskId().getExecutionBlockId(); InetSocketAddress workerAddress = getQuery().getStage(ebId).getAssignedWorkerMap().get(workerId); try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); CallFuture<PrimitiveProtos.BoolProto> callFuture = new CallFuture<>(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); if(!callFuture.get().getValue()){ getEventHandler().handle( new TaskFatalErrorEvent(taskAttemptId, new TajoInternalError("Can't kill task :" + taskAttemptId))); } } catch (Exception e) { /* Node RPC failure */ LOG.error(e.getMessage(), e); getEventHandler().handle(new TaskFatalErrorEvent(taskAttemptId, e)); } }
/** * It sends a kill RPC request to a corresponding worker. * * @param workerId worker unique Id. * @param taskAttemptId The TaskAttemptId to be killed. */ protected void killTaskAttempt(int workerId, TaskAttemptId taskAttemptId) { NettyClientBase tajoWorkerRpc; ExecutionBlockId ebId = taskAttemptId.getTaskId().getExecutionBlockId(); InetSocketAddress workerAddress = getQuery().getStage(ebId).getAssignedWorkerMap().get(workerId); try { tajoWorkerRpc = RpcClientManager.getInstance().getClient(workerAddress, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); CallFuture<PrimitiveProtos.BoolProto> callFuture = new CallFuture<PrimitiveProtos.BoolProto>(); tajoWorkerRpcClient.killTaskAttempt(null, taskAttemptId.getProto(), callFuture); if(!callFuture.get().getValue()){ getEventHandler().handle( new TaskFatalErrorEvent(taskAttemptId, new TajoInternalError("Can't kill task :" + taskAttemptId))); } } catch (Exception e) { /* Node RPC failure */ LOG.error(e.getMessage(), e); getEventHandler().handle(new TaskFatalErrorEvent(taskAttemptId, e)); } }
@Override public void statusUpdate(RpcController controller, TaskStatusProto request, RpcCallback<PrimitiveProtos.NullProto> done) { QueryId queryId = new QueryId(request.getId().getTaskId().getExecutionBlockId().getQueryId()); TaskAttemptId attemptId = new TaskAttemptId(request.getId()); QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(queryId); if (queryMasterTask != null) { Stage sq = queryMasterTask.getQuery().getStage(attemptId.getTaskId().getExecutionBlockId()); Task task = sq.getTask(attemptId.getTaskId()); TaskAttempt attempt = task.getAttempt(attemptId.getId()); if(LOG.isDebugEnabled()){ LOG.debug(String.format("Task State: %s, Attempt State: %s", task.getState().name(), attempt.getState().name())); } if (request.getState() == TajoProtos.TaskAttemptState.TA_KILLED) { LOG.warn(attemptId + " Killed"); attempt.handle( new TaskAttemptEvent(new TaskAttemptId(request.getId()), TaskAttemptEventType.TA_LOCAL_KILLED)); } else { queryMasterTask.getEventHandler().handle( new TaskAttemptStatusUpdateEvent(new TaskAttemptId(request.getId()), request)); } } done.run(TajoWorker.NULL_PROTO); }
@Override public void statusUpdate(RpcController controller, TaskStatusProto request, RpcCallback<PrimitiveProtos.NullProto> done) { QueryId queryId = new QueryId(request.getId().getTaskId().getExecutionBlockId().getQueryId()); TaskAttemptId attemptId = new TaskAttemptId(request.getId()); QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask(queryId); if (queryMasterTask != null) { Stage sq = queryMasterTask.getQuery().getStage(attemptId.getTaskId().getExecutionBlockId()); Task task = sq.getTask(attemptId.getTaskId()); TaskAttempt attempt = task.getAttempt(attemptId.getId()); if(LOG.isDebugEnabled()){ LOG.debug(String.format("Task State: %s, Attempt State: %s", task.getState().name(), attempt.getState().name())); } if (request.getState() == TajoProtos.TaskAttemptState.TA_KILLED) { LOG.warn(attemptId + " Killed"); attempt.handle( new TaskAttemptEvent(new TaskAttemptId(request.getId()), TaskAttemptEventType.TA_LOCAL_KILLED)); } else { queryMasterTask.getEventHandler().handle( new TaskAttemptStatusUpdateEvent(new TaskAttemptId(request.getId()), request)); } } done.run(TajoWorker.NULL_PROTO); }
@Override public void getExecutionBlockContext(RpcController controller, ExecutionBlockContextRequest request, RpcCallback<ExecutionBlockContextResponse> done) { QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask( new QueryId(request.getExecutionBlockId().getQueryId())); if (queryMasterTask != null) { Stage stage = queryMasterTask.getQuery().getStage(new ExecutionBlockId(request.getExecutionBlockId())); // first request with starting ExecutionBlock PlanProto.ShuffleType shuffleType = stage.getDataChannel().getShuffleType(); ExecutionBlockContextResponse.Builder ebRequestProto = ExecutionBlockContextResponse.newBuilder(); ebRequestProto.setExecutionBlockId(request.getExecutionBlockId()) .setQueryContext(stage.getContext().getQueryContext().getProto()) .setQueryOutputPath(stage.getContext().getStagingDir().toString()) .setPlanJson(CoreGsonHelper.toJson(stage.getBlock().getPlan(), LogicalNode.class)) .setShuffleType(shuffleType); //Set assigned worker to stage if (!stage.getAssignedWorkerMap().containsKey(request.getWorker().getId())) { stage.getAssignedWorkerMap().put(request.getWorker().getId(), NetUtils.createSocketAddr(request.getWorker().getHost(), request.getWorker().getPeerRpcPort())); } done.run(ebRequestProto.build()); } else { controller.setFailed("Can't find query. request: " + request); } }
@Override public void getExecutionBlockContext(RpcController controller, ExecutionBlockContextRequest request, RpcCallback<ExecutionBlockContextResponse> done) { QueryMasterTask queryMasterTask = queryMaster.getQueryMasterTask( new QueryId(request.getExecutionBlockId().getQueryId())); if (queryMasterTask != null) { Stage stage = queryMasterTask.getQuery().getStage(new ExecutionBlockId(request.getExecutionBlockId())); // first request with starting ExecutionBlock PlanProto.ShuffleType shuffleType = stage.getDataChannel().getShuffleType(); ExecutionBlockContextResponse.Builder ebRequestProto = ExecutionBlockContextResponse.newBuilder(); ebRequestProto.setExecutionBlockId(request.getExecutionBlockId()) .setQueryContext(stage.getContext().getQueryContext().getProto()) .setQueryOutputPath(stage.getContext().getStagingDir().toString()) .setPlanJson(CoreGsonHelper.toJson(stage.getBlock().getPlan(), LogicalNode.class)) .setShuffleType(shuffleType); //Set assigned worker to stage if (!stage.getAssignedWorkerMap().containsKey(request.getWorker().getId())) { stage.getAssignedWorkerMap().put(request.getWorker().getId(), NetUtils.createSocketAddr(request.getWorker().getHost(), request.getWorker().getPeerRpcPort())); } done.run(ebRequestProto.build()); } else { controller.setFailed("Can't find query. request: " + request); } }
TajoClientUtil.waitCompletion(client, queryId); MasterPlan plan = queryMasterTask.getQuery().getPlan(); ExecutionBlock rootEB = plan.getRoot(); assertEquals(1, plan.getChildCount(rootEB.getId()));
TajoClientUtil.waitCompletion(client, queryId); MasterPlan plan = queryMasterTask.getQuery().getPlan(); ExecutionBlock rootEB = plan.getRoot();