private boolean hasStagePriorAttemptSuccess(SparkStage stage) { int stageAttemptId = Integer.parseInt(stage.getTags().get(SparkJobTagName.SPARK_STAGE_ATTEMPT_ID.toString())); for (int i = 0; i < stageAttemptId; i++) { SparkStage previousStage = stages.get(this.generateStageKey( stage.getTags().get(SparkJobTagName.SPARK_SATGE_ID.toString()), Integer.toString(i))); if (previousStage.getStatus().equalsIgnoreCase(SparkEntityConstant.SparkStageStatus.COMPLETE.toString())) { return true; } } return false; }
public SparkApplicationParser(SparkRunningJobAppConfig.EagleServiceConfig eagleServiceConfig, SparkRunningJobAppConfig.EndpointConfig endpointConfig, SparkRunningJobAppConfig.JobExtractorConfig jobExtractorConfig, AppInfo app, Map<String, SparkAppEntity> sparkApp, SparkRunningJobManager sparkRunningJobManager, ResourceFetcher rmResourceFetcher) { this.sparkAppEntityCreationHandler = new SparkAppEntityCreationHandler(eagleServiceConfig); this.endpointConfig = endpointConfig; this.app = app; this.sparkJobConfigs = new HashMap<>(); this.stagesTime = new HashMap<>(); this.completeStages = new HashSet<>(); this.sparkAppEntityMap = sparkApp; if (this.sparkAppEntityMap == null) { this.sparkAppEntityMap = new HashMap<>(); } this.rmResourceFetcher = rmResourceFetcher; this.currentAttempt = 1; this.first = true; this.hdfsConf = new Configuration(); for (Map.Entry<String, String> entry : endpointConfig.hdfs.entrySet()) { this.hdfsConf.set(entry.getKey(), entry.getValue()); LOG.info("conf key {}, conf value {}", entry.getKey(), entry.getValue()); } this.hdfsConf.setBoolean("fs.hdfs.impl.disable.cache", true); this.commonTags.put(SparkJobTagName.SITE.toString(), jobExtractorConfig.site); this.commonTags.put(SparkJobTagName.SPARK_USER.toString(), app.getUser()); this.commonTags.put(SparkJobTagName.SPARK_QUEUE.toString(), app.getQueue()); this.parserStatus = ParserStatus.FINISHED; this.sparkRunningJobManager = sparkRunningJobManager; }
@Override public void read(InputStream is) throws Exception { Map<String, String> baseTags = new HashMap<>(); baseTags.put(SparkJobTagName.SITE.toString(), site); baseTags.put(SparkJobTagName.SPARK_QUEUE.toString(), app.getQueue()); JHFParserBase parser = new JHFSparkParser(new JHFSparkEventReader(config, baseTags, this.app)); parser.parse(is); }
private void initiateStage(int jobId, int stageId, int stageAttemptId, String name, int numTasks) { SparkStage stage = new SparkStage(); stage.setTags(new HashMap<>(this.app.getTags())); stage.setTimestamp(app.getTimestamp()); stage.getTags().put(SparkJobTagName.SPARK_JOB_ID.toString(), Integer.toString(jobId)); stage.getTags().put(SparkJobTagName.SPARK_SATGE_ID.toString(), Integer.toString(stageId)); stage.getTags().put(SparkJobTagName.SPARK_STAGE_ATTEMPT_ID.toString(), Integer.toString(stageAttemptId)); stage.setName(name); stage.setNumActiveTasks(0); stage.setNumTasks(numTasks); stage.setSchedulingPool(this.app.getConfig().getConfig().get("spark.scheduler.pool") == null ? "default" : this.app.getConfig().getConfig().get("spark.scheduler.pool")); String stageKey = this.generateStageKey(Integer.toString(stageId), Integer.toString(stageAttemptId)); stages.put(stageKey, stage); this.jobStageMap.get(jobId).add(stageKey); }
private SparkExecutor initiateExecutor(String executorID, long startTime) { if (!executors.containsKey(executorID)) { SparkExecutor executor = new SparkExecutor(); executor.setTags(new HashMap<>(this.app.getTags())); executor.getTags().put(SparkJobTagName.SPARK_EXECUTOR_ID.toString(), executorID); executor.setStartTime(startTime); executor.setTimestamp(app.getTimestamp()); this.executors.put(executorID, executor); } return this.executors.get(executorID); }
private void handleStageSubmit(JSONObject event) { JSONObject stageInfo = JSONUtils.getJSONObject(event, "Stage Info"); int stageId = JSONUtils.getInt(stageInfo, "Stage ID"); int stageAttemptId = JSONUtils.getInt(stageInfo, "Stage Attempt ID"); String key = this.generateStageKey(Integer.toString(stageId), Integer.toString(stageAttemptId)); stageTaskStatusMap.put(key, new HashMap<Integer, Boolean>()); if (!stages.containsKey(key)) { //may be further attempt for one stage String baseAttempt = this.generateStageKey(Integer.toString(stageId), "0"); if (stages.containsKey(baseAttempt)) { SparkStage stage = stages.get(baseAttempt); String jobId = stage.getTags().get(SparkJobTagName.SPARK_JOB_ID.toString()); String stageName = JSONUtils.getString(event, "Stage Name"); int numTasks = JSONUtils.getInt(stageInfo, "Number of Tasks"); this.initiateStage(Integer.parseInt(jobId), stageId, stageAttemptId, stageName, numTasks); } } }
private void handleAppStarted(JSONObject event) { //need update all entities tag before app start List<TaggedLogAPIEntity> entities = new ArrayList<TaggedLogAPIEntity>(); entities.addAll(this.executors.values()); entities.add(this.app); long appStartTime = JSONUtils.getLong(event, "Timestamp", lastEventTime); for (TaggedLogAPIEntity entity : entities) { entity.getTags().put(SparkJobTagName.SPARK_APP_ID.toString(), JSONUtils.getString(event, "App ID")); entity.getTags().put(SparkJobTagName.SPARK_APP_NAME.toString(), JSONUtils.getString(event, "App Name")); // In yarn-client mode, attemptId is not available in the log, so we set attemptId = 1. String attemptId = isClientMode(this.app.getConfig()) ? "1" : JSONUtils.getString(event, "App Attempt ID"); entity.getTags().put(SparkJobTagName.SPARK_APP_ATTEMPT_ID.toString(), attemptId); // the second argument of getNormalizeName() is changed to null because the original code contains sensitive text // original second argument looks like: this.app.getConfig().getConfig().get("xxx"), "xxx" is the sensitive text entity.getTags().put(SparkJobTagName.SPARK_APP_NORM_NAME.toString(), this.getNormalizedName(JSONUtils.getString(event, "App Name"), null)); entity.getTags().put(SparkJobTagName.SPARK_USER.toString(), JSONUtils.getString(event, "User")); entity.setTimestamp(appStartTime); } this.app.setStartTime(appStartTime); this.lastEventTime = appStartTime; }
private SparkTask initializeTask(JSONObject event) { SparkTask task = new SparkTask(); task.setTags(new HashMap<>(this.app.getTags())); task.setTimestamp(app.getTimestamp()); task.getTags().put(SparkJobTagName.SPARK_SATGE_ID.toString(), Long.toString(JSONUtils.getLong(event, "Stage ID"))); task.getTags().put(SparkJobTagName.SPARK_STAGE_ATTEMPT_ID.toString(), Long.toString(JSONUtils.getLong(event, "Stage Attempt ID"))); JSONObject taskInfo = JSONUtils.getJSONObject(event, "Task Info"); long taskId = JSONUtils.getLong(taskInfo, "Task ID"); task.setTaskId(taskId); task.getTags().put(SparkJobTagName.SPARK_TASK_INDEX.toString(), Long.toString(JSONUtils.getLong(taskInfo, "Index"))); task.getTags().put(SparkJobTagName.SPARK_TASK_ATTEMPT_ID.toString(), Integer.toString(JSONUtils.getInt(taskInfo, "Attempt"))); long launchTime = JSONUtils.getLong(taskInfo, "Launch Time", lastEventTime); this.lastEventTime = launchTime; if (taskId == 0) { this.setFirstTaskLaunchTime(launchTime); } task.setLaunchTime(launchTime); task.setExecutorId(JSONUtils.getString(taskInfo, "Executor ID")); task.setHost(JSONUtils.getString(taskInfo, "Host")); task.setTaskLocality(JSONUtils.getString(taskInfo, "Locality")); task.setSpeculative(JSONUtils.getBoolean(taskInfo, "Speculative")); tasks.put(task.getTaskId(), task); return task; }
private void aggregateToJob(SparkStage stage) { int jobId = Integer.parseInt(stage.getTags().get(SparkJobTagName.SPARK_JOB_ID.toString())); SparkJob job = jobs.get(jobId); job.setNumCompletedTasks(job.getNumCompletedTasks() + stage.getNumCompletedTasks()); job.setNumFailedTasks(job.getNumFailedTasks() + stage.getNumFailedTasks()); job.setNumTask(job.getNumTask() + stage.getNumTasks()); if (stage.getStatus().equalsIgnoreCase(SparkEntityConstant.SparkStageStatus.COMPLETE.toString())) { //if multiple attempts succeed, just count one if (!hasStagePriorAttemptSuccess(stage)) { job.setNumCompletedStages(job.getNumCompletedStages() + 1); } } else { job.setNumFailedStages(job.getNumFailedStages() + 1); } }
int lastSavedAttempt = 1; if (sparkAppEntityMap.containsKey(id)) { lastSavedAttempt = Integer.parseInt(sparkAppEntityMap.get(id).getTags().get(SparkJobTagName.SPARK_APP_ATTEMPT_ID.toString())); commonTags.put(SparkJobTagName.SPARK_APP_NAME.toString(), sparkApplication.getName()); commonTags.put(SparkJobTagName.SPARK_APP_ATTEMPT_ID.toString(), "" + j); commonTags.put(SparkJobTagName.SPARK_APP_ID.toString(), id); SparkAppEntity attemptEntity = new SparkAppEntity(); attemptEntity.setTags(new HashMap<>(commonTags));
private void handleJobStart(JSONObject event) { SparkJob job = new SparkJob(); job.setTags(new HashMap<>(this.app.getTags())); job.setTimestamp(app.getTimestamp()); int jobId = JSONUtils.getInt(event, "Job ID"); job.getTags().put(SparkJobTagName.SPARK_JOB_ID.toString(), Integer.toString(jobId)); long submissionTime = JSONUtils.getLong(event, "Submission Time", lastEventTime); job.setSubmissionTime(submissionTime); this.lastEventTime = submissionTime; //for complete application, no active stages/tasks job.setNumActiveStages(0); job.setNumActiveTasks(0); this.jobs.put(jobId, job); this.jobStageMap.put(jobId, new HashSet<String>()); JSONArray stages = JSONUtils.getJSONArray(event, "Stage Infos"); int stagesSize = (stages == null ? 0 : stages.size()); job.setNumStages(stagesSize); for (int i = 0; i < stagesSize; i++) { JSONObject stageInfo = (JSONObject) stages.get(i); int stageId = JSONUtils.getInt(stageInfo, "Stage ID"); int stageAttemptId = JSONUtils.getInt(stageInfo, "Stage Attempt ID"); String stageName = JSONUtils.getString(stageInfo, "Stage Name"); int numTasks = JSONUtils.getInt(stageInfo, "Number of Tasks"); this.initiateStage(jobId, stageId, stageAttemptId, stageName, numTasks); } }
int jobId = Integer.parseInt(stage.getTags().get(SparkJobTagName.SPARK_JOB_ID.toString())); if (stage.getSubmitTime() == 0 || stage.getCompleteTime() == 0) { SparkJob job = this.jobs.get(jobId); needStoreStages.add(stage); String stageId = stage.getTags().get(SparkJobTagName.SPARK_SATGE_ID.toString()); String stageAttemptId = stage.getTags().get(SparkJobTagName.SPARK_STAGE_ATTEMPT_ID.toString()); this.jobStageMap.get(jobId).remove(this.generateStageKey(stageId, stageAttemptId)); String executorID = executor.getTags().get(SparkJobTagName.SPARK_EXECUTOR_ID.toString()); if (executorID.equalsIgnoreCase("driver")) { executor.setExecMemoryBytes(driverMemory);
private void aggregateToStage(SparkTask task) { String stageId = task.getTags().get(SparkJobTagName.SPARK_SATGE_ID.toString()); String stageAttemptId = task.getTags().get(SparkJobTagName.SPARK_STAGE_ATTEMPT_ID.toString()); String key = this.generateStageKey(stageId, stageAttemptId); SparkStage stage = stages.get(key); Integer taskIndex = Integer.parseInt(task.getTags().get(SparkJobTagName.SPARK_TASK_INDEX.toString())); if (stageTaskStatusMap.get(key).containsKey(taskIndex)) {
SparkTaskEntity taskEntity = new SparkTaskEntity(); taskEntity.setTags(new HashMap<>(stageEntity.getTags())); taskEntity.getTags().put(SparkJobTagName.SPARK_TASK_ATTEMPT_ID.toString(), task.getAttempt() + ""); taskEntity.getTags().put(SparkJobTagName.SPARK_TASK_INDEX.toString(), task.getIndex() + ""); taskEntity.setTaskId(task.getTaskId()); taskEntity.setLaunchTime(Utils.dateTimeToLong(task.getLaunchTime()));