ShuffleServerMetrics shuffleMetrics = (ShuffleServerMetrics) context.getAttribute("shuffleServerMetrics"); shuffleMetrics.setHttpQueueLen(tracker.getJettyQueueSize()); long startTime = 0; try { shuffleMetrics.serverHandlerBusy(); if(ClientTraceLog.isInfoEnabled()) startTime = System.nanoTime(); rem -= len; try { shuffleMetrics.outputBytes(len); outStream.write(buffer, 0, len); outStream.flush(); shuffleMetrics.failedOutput(); throw eof; shuffleMetrics.failedOutput(); throw ie; } finally { shuffleMetrics.serverHandlerFree(); if (ClientTraceLog.isInfoEnabled()) { ClientTraceLog.info(String.format(MR_CLIENTTRACE_FORMAT, shuffleMetrics.successOutput();
shuffleMetrics.serverHandlerBusy(); outStream = response.getOutputStream(); JobConf conf = (JobConf) context.getAttribute("conf"); while (len > 0) { try { shuffleMetrics.outputBytes(len); outStream.write(buffer, 0, len); outStream.flush(); shuffleMetrics.failedOutput(); throw ie; } finally { checksumInputStream.close(); shuffleMetrics.serverHandlerFree(); if (ClientTraceLog.isInfoEnabled()) { ClientTraceLog.info(String.format(MR_CLIENTTRACE_FORMAT, shuffleMetrics.successOutput();
rem -= len; try { shuffleMetrics.outputBytes(len);
shuffleMetrics.failedOutput(); sendError(ctx, e.getMessage(), INTERNAL_SERVER_ERROR); return;
ChannelFuture writeFuture = ch.write(partition); writeFuture.addListener(new ChanneFutureListenerMetrics(partition)); shuffleMetrics.outputBytes(info.partLength); // optimistic LOG.info("Sending out " + info.partLength + " bytes for reduce: " + reduce + " from map: " + mapId + " given " +
shuffleMetrics.serverHandlerBusy(); response.setContentType("application/octet-stream"); log.warn(errorMsg, ie); response.sendError(HttpServletResponse.SC_GONE, errorMsg); shuffleMetrics.failedOutput(); throw ie; } finally { shuffleMetrics.serverHandlerFree(); shuffleMetrics.successOutput(); long timeElapsed = (System.currentTimeMillis()-start); LOG.info("Shuffled " + numMaps
httpPort == 0, conf); workerThreads = conf.getInt("tasktracker.http.threads", 40); this.shuffleServerMetrics = new ShuffleServerMetrics(conf); server.setThreads(1, workerThreads);
httpPort == 0, conf, aclsManager.getAdminsAcl()); workerThreads = conf.getInt(TT_HTTP_THREADS, 40); this.shuffleServerMetrics = new ShuffleServerMetrics(conf); server.setThreads(1, workerThreads);
@Override public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception { Channel ch = e.getChannel(); Throwable cause = e.getCause(); if (cause instanceof TooLongFrameException) { sendError(ctx, BAD_REQUEST); return; } LOG.error("Shuffle error: ", cause); shuffleMetrics.failedOutput(); if (ch.isConnected()) { LOG.error("Shuffle error " + e); sendError(ctx, INTERNAL_SERVER_ERROR); } }
/** * Start with the local machine name, and the default JobTracker */ public TaskTracker(JobConf conf) throws IOException { // Default is to use netty over jetty boolean useNetty = conf.getBoolean(NETTY_MAPOUTPUT_USE, true); this.shuffleServerMetrics = new ShuffleServerMetrics(conf); if (useNetty) { initNettyMapOutputHttpServer(conf); } initHttpServer(conf, useNetty); LOG.info("Http port " + httpPort + ", netty map output http port " + nettyMapOutputHttpPort + ", use netty = " + useNetty); initJobClient(conf); initialize(conf); initializeMapEventFetcher(); }
protected void initNettyMapOutputHttpServer(JobConf conf) throws IOException { int nettyHttpPort = conf.getInt(NETTY_MAPOUTPUT_HTTP_PORT, 0); NettyMapOutputAttributes attributes = new NettyMapOutputAttributes( conf, this, FileSystem.getLocal(conf), new LocalDirAllocator("mapred.local.dir"), shuffleServerMetrics); nettyMapOutputServer = new NettyMapOutputHttpServer(nettyHttpPort); nettyMapOutputServer.init(conf); shuffleServerMetrics.setNettyWorkerThreadPool( nettyMapOutputServer.getWorkerThreadPool()); HttpMapOutputPipelineFactory pipelineFactory = new HttpMapOutputPipelineFactory(attributes, nettyHttpPort); this.nettyMapOutputHttpPort = nettyMapOutputServer.start(pipelineFactory); }
@Override public void operationComplete(ChannelFuture future) throws Exception { partition.releaseExternalResources(); shuffleMetrics.successOutput(); } }