public static long getLong(JSONObject obj, String field) { return getLong(obj, field, 0L); }
private void handleAppEnd(JSONObject event) { long endTime = JSONUtils.getLong(event, "Timestamp", lastEventTime); app.setEndTime(endTime); this.lastEventTime = endTime; }
private void handleExecutorAdd(JSONObject event) { String executorID = (String) event.get("Executor ID"); long executorAddTime = JSONUtils.getLong(event, "Timestamp", lastEventTime); this.lastEventTime = executorAddTime; SparkExecutor executor = this.initiateExecutor(executorID, executorAddTime); JSONObject executorInfo = JSONUtils.getJSONObject(event, "Executor Info"); }
private void handleExecutorRemoved(JSONObject event) { String executorID = JSONUtils.getString(event, "Executor ID"); SparkExecutor executor = executors.get(executorID); long removedTime = JSONUtils.getLong(event, "Timestamp", lastEventTime); executor.setEndTime(removedTime); this.lastEventTime = removedTime; }
private void handleBlockManagerAdd(JSONObject event) { long maxMemory = JSONUtils.getLong(event, "Maximum Memory"); long timestamp = JSONUtils.getLong(event, "Timestamp", lastEventTime); this.lastEventTime = timestamp; JSONObject blockInfo = JSONUtils.getJSONObject(event, "Block Manager ID"); String executorID = JSONUtils.getString(blockInfo, "Executor ID"); String hostAndPort = JSONUtils.getString(blockInfo, "Host") + ":" + JSONUtils.getLong(blockInfo, "Port"); SparkExecutor executor = this.initiateExecutor(executorID, timestamp); executor.setMaxMemory(maxMemory); executor.setHostPort(hostAndPort); }
private void handleTaskEnd(JSONObject event) { JSONObject taskInfo = JSONUtils.getJSONObject(event, "Task Info"); long taskId = JSONUtils.getLong(taskInfo, "Task ID"); SparkTask task = tasks.get(taskId); if (task == null) { JSONObject taskMetrics = JSONUtils.getJSONObject(event, "Task Metrics"); if (null != taskMetrics) { task.setExecutorDeserializeTime(JSONUtils.getLong(taskMetrics, "Executor Deserialize Time", lastEventTime)); task.setExecutorRunTime(JSONUtils.getLong(taskMetrics, "Executor Run Time", lastEventTime)); task.setJvmGcTime(JSONUtils.getLong(taskMetrics, "JVM GC Time", lastEventTime)); task.setResultSize(JSONUtils.getLong(taskMetrics, "Result Size")); task.setResultSerializationTime(JSONUtils.getLong(taskMetrics, "Result Serialization Time", lastEventTime)); task.setMemoryBytesSpilled(JSONUtils.getLong(taskMetrics, "Memory Bytes Spilled")); task.setDiskBytesSpilled(JSONUtils.getLong(taskMetrics, "Disk Bytes Spilled")); task.setInputBytes(JSONUtils.getLong(inputMetrics, "Bytes Read")); task.setInputRecords(JSONUtils.getLong(inputMetrics, "Records Read")); task.setOutputBytes(JSONUtils.getLong(outputMetrics, "Bytes Written")); task.setOutputRecords(JSONUtils.getLong(outputMetrics, "Records Written")); task.setShuffleWriteBytes(JSONUtils.getLong(shuffleWriteMetrics, "Shuffle Bytes Written")); task.setShuffleWriteRecords(JSONUtils.getLong(shuffleWriteMetrics, "Shuffle Records Written")); task.setShuffleReadLocalBytes(JSONUtils.getLong(shuffleReadMetrics, "Local Bytes Read")); task.setShuffleReadRemoteBytes(JSONUtils.getLong(shuffleReadMetrics, "Remote Bytes Read")); task.setShuffleReadRecords(JSONUtils.getLong(shuffleReadMetrics, "Total Records Read"));
private void handleStageComplete(JSONObject event) { JSONObject stageInfo = JSONUtils.getJSONObject(event, "Stage Info"); int stageId = JSONUtils.getInt(stageInfo, "Stage ID"); int stageAttemptId = JSONUtils.getInt(stageInfo, "Stage Attempt ID"); String key = this.generateStageKey(Integer.toString(stageId), Integer.toString(stageAttemptId)); SparkStage stage = stages.get(key); // If "Submission Time" is not available, use the "Launch Time" of "Task ID" = 0. Long submissionTime = JSONUtils.getLong(stageInfo, "Submission Time", firstTaskLaunchTime); stage.setSubmitTime(submissionTime); long completeTime = JSONUtils.getLong(stageInfo, "Completion Time", lastEventTime); stage.setCompleteTime(completeTime); this.lastEventTime = completeTime; if (stageInfo != null && stageInfo.containsKey("Failure Reason")) { stage.setStatus(SparkEntityConstant.SparkStageStatus.FAILED.toString()); } else { stage.setStatus(SparkEntityConstant.SparkStageStatus.COMPLETE.toString()); } }
private SparkTask initializeTask(JSONObject event) { SparkTask task = new SparkTask(); task.setTags(new HashMap<>(this.app.getTags())); task.setTimestamp(app.getTimestamp()); task.getTags().put(SparkJobTagName.SPARK_SATGE_ID.toString(), Long.toString(JSONUtils.getLong(event, "Stage ID"))); task.getTags().put(SparkJobTagName.SPARK_STAGE_ATTEMPT_ID.toString(), Long.toString(JSONUtils.getLong(event, "Stage Attempt ID"))); JSONObject taskInfo = JSONUtils.getJSONObject(event, "Task Info"); long taskId = JSONUtils.getLong(taskInfo, "Task ID"); task.setTaskId(taskId); task.getTags().put(SparkJobTagName.SPARK_TASK_INDEX.toString(), Long.toString(JSONUtils.getLong(taskInfo, "Index"))); task.getTags().put(SparkJobTagName.SPARK_TASK_ATTEMPT_ID.toString(), Integer.toString(JSONUtils.getInt(taskInfo, "Attempt"))); long launchTime = JSONUtils.getLong(taskInfo, "Launch Time", lastEventTime); this.lastEventTime = launchTime; if (taskId == 0) { this.setFirstTaskLaunchTime(launchTime); } task.setLaunchTime(launchTime); task.setExecutorId(JSONUtils.getString(taskInfo, "Executor ID")); task.setHost(JSONUtils.getString(taskInfo, "Host")); task.setTaskLocality(JSONUtils.getString(taskInfo, "Locality")); task.setSpeculative(JSONUtils.getBoolean(taskInfo, "Speculative")); tasks.put(task.getTaskId(), task); return task; }
private void handleJobEnd(JSONObject event) { int jobId = JSONUtils.getInt(event, "Job ID"); SparkJob job = jobs.get(jobId); long completionTime = JSONUtils.getLong(event, "Completion Time", lastEventTime); job.setCompletionTime(completionTime); this.lastEventTime = completionTime; JSONObject jobResult = JSONUtils.getJSONObject(event, "Job Result"); String result = JSONUtils.getString(jobResult, "Result"); if (result.equalsIgnoreCase("JobSucceeded")) { job.setStatus(SparkEntityConstant.SparkJobStatus.SUCCEEDED.toString()); } else { job.setStatus(SparkEntityConstant.SparkJobStatus.FAILED.toString()); } }
private void handleAppStarted(JSONObject event) { //need update all entities tag before app start List<TaggedLogAPIEntity> entities = new ArrayList<TaggedLogAPIEntity>(); entities.addAll(this.executors.values()); entities.add(this.app); long appStartTime = JSONUtils.getLong(event, "Timestamp", lastEventTime); for (TaggedLogAPIEntity entity : entities) { entity.getTags().put(SparkJobTagName.SPARK_APP_ID.toString(), JSONUtils.getString(event, "App ID")); entity.getTags().put(SparkJobTagName.SPARK_APP_NAME.toString(), JSONUtils.getString(event, "App Name")); // In yarn-client mode, attemptId is not available in the log, so we set attemptId = 1. String attemptId = isClientMode(this.app.getConfig()) ? "1" : JSONUtils.getString(event, "App Attempt ID"); entity.getTags().put(SparkJobTagName.SPARK_APP_ATTEMPT_ID.toString(), attemptId); // the second argument of getNormalizeName() is changed to null because the original code contains sensitive text // original second argument looks like: this.app.getConfig().getConfig().get("xxx"), "xxx" is the sensitive text entity.getTags().put(SparkJobTagName.SPARK_APP_NORM_NAME.toString(), this.getNormalizedName(JSONUtils.getString(event, "App Name"), null)); entity.getTags().put(SparkJobTagName.SPARK_USER.toString(), JSONUtils.getString(event, "User")); entity.setTimestamp(appStartTime); } this.app.setStartTime(appStartTime); this.lastEventTime = appStartTime; }
private void handleJobStart(JSONObject event) { SparkJob job = new SparkJob(); job.setTags(new HashMap<>(this.app.getTags())); job.setTimestamp(app.getTimestamp()); int jobId = JSONUtils.getInt(event, "Job ID"); job.getTags().put(SparkJobTagName.SPARK_JOB_ID.toString(), Integer.toString(jobId)); long submissionTime = JSONUtils.getLong(event, "Submission Time", lastEventTime); job.setSubmissionTime(submissionTime); this.lastEventTime = submissionTime; //for complete application, no active stages/tasks job.setNumActiveStages(0); job.setNumActiveTasks(0); this.jobs.put(jobId, job); this.jobStageMap.put(jobId, new HashSet<String>()); JSONArray stages = JSONUtils.getJSONArray(event, "Stage Infos"); int stagesSize = (stages == null ? 0 : stages.size()); job.setNumStages(stagesSize); for (int i = 0; i < stagesSize; i++) { JSONObject stageInfo = (JSONObject) stages.get(i); int stageId = JSONUtils.getInt(stageInfo, "Stage ID"); int stageAttemptId = JSONUtils.getInt(stageInfo, "Stage Attempt ID"); String stageName = JSONUtils.getString(stageInfo, "Stage Name"); int numTasks = JSONUtils.getInt(stageInfo, "Number of Tasks"); this.initiateStage(jobId, stageId, stageAttemptId, stageName, numTasks); } }