@Override public void abortTask(TaskAttemptContext context) throws IOException { for (Map.Entry<String, OutputJobInfo> outputJobInfoEntry : dynamicOutputJobInfo.entrySet()) { String dynKey = outputJobInfoEntry.getKey(); OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue(); LOG.info("Aborting task-attempt for " + outputJobInfo.getLocation()); baseDynamicCommitters.get(dynKey) .abortTask(dynamicContexts.get(dynKey)); } }
@Override public void commitTask(TaskAttemptContext context) throws IOException { for (Map.Entry<String, OutputJobInfo> outputJobInfoEntry : dynamicOutputJobInfo.entrySet()) { String dynKey = outputJobInfoEntry.getKey(); OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue(); LOG.info("Committing task-attempt for " + outputJobInfo.getLocation()); TaskAttemptContext dynContext = dynamicContexts.get(dynKey); OutputCommitter dynCommitter = baseDynamicCommitters.get(dynKey); if (dynCommitter.needsTaskCommit(dynContext)) { dynCommitter.commitTask(dynContext); } else { LOG.info("Skipping commitTask() for " + outputJobInfo.getLocation()); } } } });
private String generateKey(TaskAttemptContext context) throws IOException { String jobInfoString = context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO); if (StringUtils.isBlank(jobInfoString)) { // Avoid the NPE. throw new IOException("Could not retrieve OutputJobInfo for TaskAttempt " + context.getTaskAttemptID()); } OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(jobInfoString); return context.getTaskAttemptID().toString() + "@" + jobInfo.getLocation(); }
Path outputPath = new Path(jobInfo.getLocation()); FileSystem fileSys = outputPath.getFileSystem(jobContext .getConfiguration());
String location = jobInfo.getLocation();
int maxDynamicPartitions = jobInfo.getMaxDynamicPartitions(); Path loadPath = new Path(jobInfo.getLocation()); FileSystem fs = loadPath.getFileSystem(context.getConfiguration());
} else if ((dynHash == null) && Boolean.parseBoolean((String)tableDesc.getProperties().get("EXTERNAL")) && jobInfo.getLocation() != null && jobInfo.getLocation().length() > 0) { outputLocation = jobInfo.getLocation(); } else if (dynHash == null && jobInfo.getPartitionValues().size() == 0) { jobProperties.put("mapred.output.dir", jobInfo.getLocation());
if (dynamicPartitioningUsed) { if (!customDynamicLocationUsed) { src = new Path(getPartitionRootLocation(jobInfo.getLocation(), jobInfo.getTableInfo().getTable() .getPartitionKeysSize())); } else { src = new Path(jobInfo.getLocation());
new FileOutputCommitter(new Path(localJobInfo.getLocation()), currTaskContext) .getWorkPath().toString());
} else if (!dynamicPartitioningUsed && Boolean.parseBoolean((String)table.getProperty("EXTERNAL")) && jobInfo.getLocation() != null && jobInfo.getLocation().length() > 0) { String jobLocation = jobInfo.getLocation(); String finalLocn = jobLocation.replaceAll(Path.SEPARATOR + SCRATCH_DIR_NAME + "\\d\\.?\\d+",""); partPath = new Path(finalLocn);
Path src = new Path(jobInfo.getLocation()); moveTaskOutputs(fs, src, src, tblPath, false, table.isImmutable()); if (!src.equals(tblPath)) { Path src = new Path(jobInfo.getLocation()); Path dest = new Path(p.getSd().getLocation()); moveTaskOutputs(fs, src, src, dest, true, table.isImmutable());
@Override public void abortTask(TaskAttemptContext context) throws IOException { for (Map.Entry<String, OutputJobInfo> outputJobInfoEntry : dynamicOutputJobInfo.entrySet()) { String dynKey = outputJobInfoEntry.getKey(); OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue(); LOG.info("Aborting task-attempt for " + outputJobInfo.getLocation()); baseDynamicCommitters.get(dynKey) .abortTask(dynamicContexts.get(dynKey)); } }
@Override public void abortTask(TaskAttemptContext context) throws IOException { for (Map.Entry<String, OutputJobInfo> outputJobInfoEntry : dynamicOutputJobInfo.entrySet()) { String dynKey = outputJobInfoEntry.getKey(); OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue(); LOG.info("Aborting task-attempt for " + outputJobInfo.getLocation()); baseDynamicCommitters.get(dynKey) .abortTask(dynamicContexts.get(dynKey)); } }
@Override public void abortTask(TaskAttemptContext context) throws IOException { for (Map.Entry<String, OutputJobInfo> outputJobInfoEntry : dynamicOutputJobInfo.entrySet()) { String dynKey = outputJobInfoEntry.getKey(); OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue(); LOG.info("Aborting task-attempt for " + outputJobInfo.getLocation()); baseDynamicCommitters.get(dynKey) .abortTask(dynamicContexts.get(dynKey)); } }
@Override public void commitTask(TaskAttemptContext context) throws IOException { for (Map.Entry<String, OutputJobInfo> outputJobInfoEntry : dynamicOutputJobInfo.entrySet()) { String dynKey = outputJobInfoEntry.getKey(); OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue(); LOG.info("Committing task-attempt for " + outputJobInfo.getLocation()); TaskAttemptContext dynContext = dynamicContexts.get(dynKey); OutputCommitter dynCommitter = baseDynamicCommitters.get(dynKey); if (dynCommitter.needsTaskCommit(dynContext)) { dynCommitter.commitTask(dynContext); } else { LOG.info("Skipping commitTask() for " + outputJobInfo.getLocation()); } } } });
@Override public void abortTask(TaskAttemptContext context) throws IOException { for (Map.Entry<String, OutputJobInfo> outputJobInfoEntry : dynamicOutputJobInfo.entrySet()) { String dynKey = outputJobInfoEntry.getKey(); OutputJobInfo outputJobInfo = outputJobInfoEntry.getValue(); LOG.info("Aborting task-attempt for " + outputJobInfo.getLocation()); baseDynamicCommitters.get(dynKey) .abortTask(dynamicContexts.get(dynKey)); } }
private String generateKey(TaskAttemptContext context) throws IOException { String jobInfoString = context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO); if (StringUtils.isBlank(jobInfoString)) { // Avoid the NPE. throw new IOException("Could not retrieve OutputJobInfo for TaskAttempt " + context.getTaskAttemptID()); } OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(jobInfoString); return context.getTaskAttemptID().toString() + "@" + jobInfo.getLocation(); }
private String generateKey(TaskAttemptContext context) throws IOException { String jobInfoString = context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO); if (StringUtils.isBlank(jobInfoString)) { // Avoid the NPE. throw new IOException("Could not retrieve OutputJobInfo for TaskAttempt " + context.getTaskAttemptID()); } OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(jobInfoString); return context.getTaskAttemptID().toString() + "@" + jobInfo.getLocation(); }
private String generateKey(TaskAttemptContext context) throws IOException { String jobInfoString = context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO); if (StringUtils.isBlank(jobInfoString)) { // Avoid the NPE. throw new IOException("Could not retrieve OutputJobInfo for TaskAttempt " + context.getTaskAttemptID()); } OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(jobInfoString); return context.getTaskAttemptID().toString() + "@" + jobInfo.getLocation(); }
private String generateKey(TaskAttemptContext context) throws IOException { String jobInfoString = context.getConfiguration().get(HCatConstants.HCAT_KEY_OUTPUT_INFO); if (StringUtils.isBlank(jobInfoString)) { // Avoid the NPE. throw new IOException("Could not retrieve OutputJobInfo for TaskAttempt " + context.getTaskAttemptID()); } OutputJobInfo jobInfo = (OutputJobInfo) HCatUtil.deserialize(jobInfoString); return context.getTaskAttemptID().toString() + "@" + jobInfo.getLocation(); }