/** * Dumps the full job logs for a particular application to stdout. * * @param applicationId application to dump logs for */ public static void dumpFullLogs(Configuration cfg, ApplicationId applicationId) { LogCLIHelpers logCliHelpers = new LogCLIHelpers(); // TODO: Add the proper base dir settings etc... logCliHelpers.setConf(cfg); try { logCliHelpers.dumpAllContainersLogs( applicationId, cfg.get(SPYDRA_HISTORY_USERNAME_PROPERTY), System.out); } catch (IOException e) { logger.error("Failed dumping log files for application " + applicationId.toString(), e); } }
public int dumpAContainersLogs(String appId, String containerId, String nodeId, String jobOwner) throws IOException { Path remoteRootLogDir = new Path(getConf().get( YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner, try { Path qualifiedLogDir = FileContext.getFileContext(getConf()).makeQualified( remoteAppLogDir); nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), getConf()) .listStatus(remoteAppLogDir); } catch (FileNotFoundException fnf) { logDirNotExist(remoteAppLogDir.toString()); return -1; try { reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); if (dumpAContainerLogs(containerId, reader, System.out, thisNodeFile.getModificationTime()) > -1) { foundContainerLogs = true; containerLogNotFound(containerId);
@Private public int dumpAllContainersLogs(ApplicationId appId, String appOwner, PrintStream out) throws IOException { Path remoteRootLogDir = new Path(getConf().get( YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String user = appOwner; String logDirSuffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); try { Path qualifiedLogDir = FileContext.getFileContext(getConf()).makeQualified(remoteAppLogDir); nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), getConf()).listStatus(remoteAppLogDir); } catch (FileNotFoundException fnf) { logDirNotExist(remoteAppLogDir.toString()); return -1; .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); try { emptyLogDir(remoteAppLogDir.toString()); return -1;
@Private public int dumpAContainerLogsForLogTypeWithoutNodeId( ContainerLogsRequest options) throws IOException { LogAggregationFileController fc = null; try { fc = this.getFileController( options.getAppId(), options.getAppOwner()); } catch (IOException ex) { System.err.println(ex); } boolean foundAnyLogs = false; if (fc != null) { foundAnyLogs = fc.readAggregatedLogs(options, null); } if (!foundAnyLogs) { containerLogNotFound(options.getContainerId()); return -1; } return 0; }
TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid); LogParams logParams = cluster.getLogParams(jobID, taskAttemptID); LogCLIHelpers logDumper = new LogCLIHelpers(); logDumper.setConf(getConf()); exitCode = logDumper.dumpAContainersLogs(logParams.getApplicationId(), logParams.getContainerId(), logParams.getNodeId(), logParams.getOwner());
LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); } else if ((containerIdStr == null && nodeAddress != null) || (containerIdStr != null && nodeAddress == null)) { } else { resultCode = logCliHelper.dumpAContainersLogs(appIdStr, containerIdStr, nodeAddress, appOwner);
LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); logs = Arrays.asList(logFiles); return logCliHelper.dumpAContainersLogsForALogType(appIdStr, containerIdStr, nodeAddress, appOwner, logs); if (nodeAddress == null) { resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); } else { System.out.println("Should at least provide ContainerId!");
webServiceClient.addFilter(retryFilter); LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf());
LogAggregationFileController fileFormat = null; try { fileFormat = getFileController(appId, appOwner); } catch (Exception ex) { err.println(ex.getMessage()); fileFormat.getRemoteRootLogDirSuffix()); } catch (FileNotFoundException fnf) { logDirNotExist(LogAggregationUtils.getRemoteAppLogDir( conf, appId, appOwner).toString()); } catch (AccessControlException | AccessDeniedException ace) { logDirNoAccessPermission(LogAggregationUtils.getRemoteAppLogDir( conf, appId, appOwner).toString(), appOwner, ace.getMessage());
} else { resultCode = logCliHelper.dumpAllContainersLogs(newOptions);
@Private public int dumpAllContainersLogs(ContainerLogsRequest options) throws IOException { LogAggregationFileController fc = null; try { fc = this.getFileController( options.getAppId(), options.getAppOwner()); } catch (IOException ex) { System.err.println(ex); } boolean foundAnyLogs = false; if (fc != null) { foundAnyLogs = fc.readAggregatedLogs(options, null); } if (!foundAnyLogs) { emptyLogDir(LogAggregationUtils.getRemoteAppLogDir( conf, options.getAppId(), options.getAppOwner()) .toString()); return -1; } return 0; }
TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid); LogParams logParams = cluster.getLogParams(jobID, taskAttemptID); LogCLIHelpers logDumper = new LogCLIHelpers(); logDumper.setConf(getConf()); exitCode = logDumper.dumpAContainersLogs(logParams.getApplicationId(), logParams.getContainerId(), logParams.getNodeId(), logParams.getOwner());
LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); resultCode = logCliHelper.dumpAllContainersLogs(appId, appOwner, System.out); } else if ((containerIdStr == null && nodeAddress != null) || (containerIdStr != null && nodeAddress == null)) { } else { resultCode = logCliHelper.dumpAContainersLogs(appIdStr, containerIdStr, nodeAddress, appOwner);
@Private @VisibleForTesting public int dumpAContainerLogsForLogType(ContainerLogsRequest options, boolean outputFailure) throws IOException { LogAggregationFileController fc = null; try { fc = this.getFileController( options.getAppId(), options.getAppOwner()); } catch (IOException ex) { System.err.println(ex); } boolean foundAnyLogs = false; if (fc != null) { foundAnyLogs = fc.readAggregatedLogs(options, null); } if (!foundAnyLogs) { if (outputFailure) { containerLogNotFound(options.getContainerId()); } return -1; } return 0; }
TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid); LogParams logParams = cluster.getLogParams(jobID, taskAttemptID); LogCLIHelpers logDumper = new LogCLIHelpers(); logDumper.setConf(getConf()); exitCode = logDumper.dumpAContainersLogs(logParams.getApplicationId(), logParams.getContainerId(), logParams.getNodeId(), logParams.getOwner());
public int dumpAContainersLogs(String appId, String containerId, String nodeId, String jobOwner) throws IOException { Path remoteRootLogDir = new Path(getConf().get( YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir( remoteRootLogDir, ConverterUtils.toApplicationId(appId), jobOwner, try { Path qualifiedLogDir = FileContext.getFileContext(getConf()).makeQualified( remoteAppLogDir); nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), getConf()) .listStatus(remoteAppLogDir); } catch (FileNotFoundException fnf) { logDirNotExist(remoteAppLogDir.toString()); return -1; try { reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); if (dumpAContainerLogs(containerId, reader, System.out, thisNodeFile.getModificationTime()) > -1) { foundContainerLogs = true; containerLogNotFound(containerId);
@Private public int dumpAllContainersLogs(ApplicationId appId, String appOwner, PrintStream out) throws IOException { Path remoteRootLogDir = new Path(getConf().get( YarnConfiguration.NM_REMOTE_APP_LOG_DIR, YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); String user = appOwner; String logDirSuffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(getConf()); try { Path qualifiedLogDir = FileContext.getFileContext(getConf()).makeQualified(remoteAppLogDir); nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), getConf()).listStatus(remoteAppLogDir); } catch (FileNotFoundException fnf) { logDirNotExist(remoteAppLogDir.toString()); return -1; .endsWith(LogAggregationUtils.TMP_FILE_SUFFIX)) { AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(getConf(), thisNodeFile.getPath()); try { emptyLogDir(remoteAppLogDir.toString()); return -1;
TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid); LogParams logParams = cluster.getLogParams(jobID, taskAttemptID); LogCLIHelpers logDumper = new LogCLIHelpers(); logDumper.setConf(getConf()); exitCode = logDumper.dumpAContainersLogs(logParams.getApplicationId(), logParams.getContainerId(), logParams.getNodeId(), logParams.getOwner());