request.getFragmentNumber(), request.getAttemptNumber()); String fragmentIdString = attemptId.toString(); if (LOG.isInfoEnabled()) { HistoryLogger.logFragmentStart(qIdProto.getApplicationIdString(), request.getContainerIdString(), localAddress.get().getHostName(), constructUniqueQueryId(vertex.getHiveQueryId(), qIdProto.getDagIndex()), qIdProto.getDagIndex(), vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber()); request.getCredentialsBinary().toByteArray()); LlapNodeId amNodeId = LlapNodeId.getInstance(request.getAmHost(), request.getAmPort()); QueryFragmentInfo fragmentInfo = queryTracker.registerFragment( queryIdentifier, qIdProto.getApplicationIdString(), dagId, vertex.getDagName(), vertex.getHiveQueryId(), dagIdentifier, vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(), vertex.getUser(), vertex, jobToken, fragmentIdString, tokenInfo, amNodeId); boolean isGuaranteed = request.hasIsGuaranteed() && request.getIsGuaranteed();
hash = (19 * hash) + getDescriptorForType().hashCode(); if (hasWorkSpec()) { hash = (37 * hash) + WORK_SPEC_FIELD_NUMBER; hash = (53 * hash) + getWorkSpec().hashCode(); if (hasWorkSpecSignature()) { hash = (37 * hash) + WORK_SPEC_SIGNATURE_FIELD_NUMBER; hash = (53 * hash) + getWorkSpecSignature().hashCode(); if (hasFragmentNumber()) { hash = (37 * hash) + FRAGMENT_NUMBER_FIELD_NUMBER; hash = (53 * hash) + getFragmentNumber(); if (hasAttemptNumber()) { hash = (37 * hash) + ATTEMPT_NUMBER_FIELD_NUMBER; hash = (53 * hash) + getAttemptNumber(); if (hasContainerIdString()) { hash = (37 * hash) + CONTAINER_ID_STRING_FIELD_NUMBER; hash = (53 * hash) + getContainerIdString().hashCode(); if (hasAmHost()) { hash = (37 * hash) + AM_HOST_FIELD_NUMBER; hash = (53 * hash) + getAmHost().hashCode(); if (hasAmPort()) { hash = (37 * hash) + AM_PORT_FIELD_NUMBER; hash = (53 * hash) + getAmPort();
runtimeInfo.setNumSelfAndUpstreamCompletedTasks(0); SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder();
this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort(), fragmentInfo.getQueryInfo().getQueryIdentifier(), ta); taskOwner.addToken(jobToken); final InetSocketAddress address = NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort()); SecurityUtil.setTokenService(jobToken, address); NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort()); umbilical = taskOwner.doAs(new PrivilegedExceptionAction<LlapTaskUmbilicalProtocol>() { @Override confParams.amMaxEventsPerHeartbeat, new AtomicLong(0), request.getContainerIdString(), fragmentId, initialEvent, FileSystem.closeAllForUGI(fsTaskUgi); fragmentInfo.getQueryInfo().returnUmbilicalUgi(taskOwner); LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" + runtimeWatch.stop().elapsed(TimeUnit.MILLISECONDS)); if (LOG.isDebugEnabled()) {
request.getFragmentNumber(), request.getAttemptNumber()); String fragmentIdString = attemptId.toString(); if (LOG.isInfoEnabled()) { HistoryLogger.logFragmentStart(qIdProto.getApplicationIdString(), request.getContainerIdString(), localAddress.get().getHostName(), constructUniqueQueryId(vertex.getHiveQueryId(), qIdProto.getDagIndex()), qIdProto.getDagIndex(), vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber()); byte[] tokenBytes = request.getCredentialsBinary().toByteArray(); dib.reset(tokenBytes, tokenBytes.length); credentials.readTokenStorageStream(dib); LlapNodeId amNodeId = LlapNodeId.getInstance(request.getAmHost(), request.getAmPort()); QueryFragmentInfo fragmentInfo = queryTracker.registerFragment( queryIdentifier, qIdProto.getApplicationIdString(), dagId, vertex.getDagName(), vertex.getHiveQueryId(), dagIdentifier, vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(), vertex.getUser(), vertex, jobToken, fragmentIdString, tokenInfo, amNodeId); boolean isGuaranteed = request.hasIsGuaranteed() && request.getIsGuaranteed();
request.getFragmentNumber(), request.getAttemptNumber()).toString(); OutputStream socketStream = socket.getOutputStream(); LlapOutputSocketInitMessage.Builder builder =
runtimeInfo.setNumSelfAndUpstreamCompletedTasks(0); SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder();
public static String stringifySubmitRequest( SubmitWorkRequestProto request, SignableVertexSpec vertex) { StringBuilder sb = new StringBuilder(); sb.append("am_details=").append(request.getAmHost()).append(":").append(request.getAmPort()); sb.append(", taskInfo=").append(" fragment ") .append(request.getFragmentNumber()).append(" attempt ").append(request.getAttemptNumber()); sb.append(", user=").append(vertex.getUser()); sb.append(", queryId=").append(vertex.getHiveQueryId()); sb.append(", appIdString=").append(vertex.getQueryIdentifier().getApplicationIdString()); sb.append(", appAttemptNum=").append(vertex.getQueryIdentifier().getAppAttemptNumber()); sb.append(", containerIdString=").append(request.getContainerIdString()); sb.append(", dagName=").append(vertex.getDagName()); sb.append(", vertexName=").append(vertex.getVertexName()); FragmentRuntimeInfo fragmentRuntimeInfo = request.getFragmentRuntimeInfo(); sb.append(", FragmentRuntimeInfo={"); sb.append("taskCount=").append(fragmentRuntimeInfo.getNumSelfAndUpstreamTasks());
String hiveQueryId) throws IOException { SubmitWorkRequestProto.Builder builder = SubmitWorkRequestProto.newBuilder(); builder.setFragmentNumber(taskSpec.getTaskAttemptID().getTaskID().getId()); builder.setAttemptNumber(taskSpec.getTaskAttemptID().getId());
/** * Submit the work for actual execution. */ public void submitWork(SubmitWorkRequestProto request, String llapHost, int llapPort) { // Register the pending events to be sent for this spec. VertexOrBinary vob = request.getWorkSpec(); assert vob.hasVertexBinary() != vob.hasVertex(); SignableVertexSpec vertex = null; try { vertex = vob.hasVertex() ? vob.getVertex() : SignableVertexSpec.parseFrom(vob.getVertexBinary()); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } QueryIdentifierProto queryIdentifierProto = vertex.getQueryIdentifier(); TezTaskAttemptID attemptId = Converters.createTaskAttemptId(queryIdentifierProto, vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); final String fragmentId = attemptId.toString(); this.requestInfo = new RequestInfo(request, queryIdentifierProto, fragmentId, llapHost, llapPort); this.tezEvents = Lists.<TezEvent>newArrayList(); registerClient(); // Send out the actual SubmitWorkRequest final LlapTaskUmbilicalExternalClient client = this; communicator.start(); submitWork(); }
private SignableVertexSpec extractVertexSpec(SubmitWorkRequestProto request, LlapTokenInfo tokenInfo) throws InvalidProtocolBufferException, IOException { VertexOrBinary vob = request.getWorkSpec(); SignableVertexSpec vertex = vob.hasVertex() ? vob.getVertex() : null; ByteString vertexBinary = vob.hasVertexBinary() ? vob.getVertexBinary() : null; if (vertexBinary != null) { if (vertex != null) { throw new IOException( "Vertex and vertexBinary in VertexOrBinary cannot be set at the same time"); } vertex = SignableVertexSpec.parseFrom(vob.getVertexBinary()); } if (tokenInfo.isSigningRequired) { checkSignature(vertex, vertexBinary, request, tokenInfo.userName); } return vertex; }
public FragmentRuntimeInfo getFragmentRuntimeInfo() { return request.getFragmentRuntimeInfo(); }
public static String getTaskIdentifierString( SubmitWorkRequestProto request, SignableVertexSpec vertex, String queryId) { StringBuilder sb = new StringBuilder(); sb.append("AppId=").append(vertex.getQueryIdentifier().getApplicationIdString()) .append(", containerId=").append(request.getContainerIdString()) .append(", QueryId=").append(queryId) .append(", Vertex=").append(vertex.getVertexName()) .append(", FragmentNum=").append(request.getFragmentNumber()) .append(", Attempt=").append(request.getAttemptNumber()); return sb.toString(); }
@Override public String toString() { return requestId + " {canFinish: " + canFinish() + ", vertexParallelism: " + vertex.getVertexParallelism() + ", selfAndUpstreamParallelism: " + request.getFragmentRuntimeInfo().getNumSelfAndUpstreamTasks() + ", selfAndUpstreamComplete: " + request.getFragmentRuntimeInfo().getNumSelfAndUpstreamCompletedTasks() + ", firstAttemptStartTime: " + getFragmentRuntimeInfo().getFirstAttemptStartTime() + ", dagStartTime:" + getFragmentRuntimeInfo().getDagStartTime() + ", withinDagPriority: " + getFragmentRuntimeInfo().getWithinDagPriority() + "}"; }
protected void logFragmentEnd(boolean success) { LOG.info("WM counters: {}", wmCounters); HistoryLogger.logFragmentEnd(vertex.getQueryIdentifier().getApplicationIdString(), request.getContainerIdString(), executionContext.getHostName(), queryId, fragmentInfo.getQueryInfo().getDagIdentifier(), vertex.getVertexName(), request.getFragmentNumber(), request.getAttemptNumber(), taskRunnerCallable.threadName, taskRunnerCallable.startTime, success); } }
this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort(), fragmentInfo.getQueryInfo().getQueryIdentifier(), ta); taskOwner.addToken(jobToken); final InetSocketAddress address = NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort()); SecurityUtil.setTokenService(jobToken, address); NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort()); umbilical = taskOwner.doAs(new PrivilegedExceptionAction<LlapTaskUmbilicalProtocol>() { @Override confParams.amMaxEventsPerHeartbeat, new AtomicLong(0), request.getContainerIdString(), fragmentId, initialEvent, FileSystem.closeAllForUGI(fsTaskUgi); fragmentInfo.getQueryInfo().returnUmbilicalUgi(taskOwner); LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" + runtimeWatch.stop().elapsed(TimeUnit.MILLISECONDS)); if (LOG.isDebugEnabled()) {
request.getFragmentNumber(), request.getAttemptNumber()).toString(); OutputStream socketStream = socket.getOutputStream(); LlapOutputSocketInitMessage.Builder builder =