/** * Configure the output storage handler * @param jobContext the job context * @throws IOException */ @SuppressWarnings("unchecked") static void configureOutputStorageHandler( JobContext jobContext) throws IOException { configureOutputStorageHandler(jobContext, (List<String>) null); }
/** * Configure the output storage handler, with allowing specification * of partvals from which it picks the dynamic partvals * @param context the job context * @param jobInfo the output job info * @param fullPartSpec * @throws IOException */ protected static void configureOutputStorageHandler( JobContext context, OutputJobInfo jobInfo, Map<String, String> fullPartSpec) throws IOException { List<String> dynamicPartKeys = jobInfo.getDynamicPartitioningKeys(); if ((dynamicPartKeys == null) || (dynamicPartKeys.isEmpty())) { configureOutputStorageHandler(context, (List<String>) null); } else { List<String> dynKeyVals = new ArrayList<String>(); for (String dynamicPartKey : dynamicPartKeys) { dynKeyVals.add(fullPartSpec.get(dynamicPartKey)); } configureOutputStorageHandler(context, dynKeyVals); } }
/** * Gets the output format instance. * @param context the job context * @return the output format instance * @throws IOException */ protected OutputFormat<WritableComparable<?>, HCatRecord> getOutputFormat(JobContext context) throws IOException { OutputJobInfo jobInfo = getJobInfo(context.getConfiguration()); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo()); // Always configure storage handler with jobproperties/jobconf before calling any methods on it configureOutputStorageHandler(context); if (storageHandler instanceof FosterStorageHandler) { return new FileOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } else { return new DefaultOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } }
/** * Configure the output storage handler * @param jobContext the job context * @throws IOException */ @SuppressWarnings("unchecked") static void configureOutputStorageHandler( JobContext jobContext) throws IOException { configureOutputStorageHandler(jobContext, (List<String>) null); }
/** * Configure the output storage handler * @param jobContext the job context * @throws IOException */ @SuppressWarnings("unchecked") static void configureOutputStorageHandler( JobContext jobContext) throws IOException { configureOutputStorageHandler(jobContext, (List<String>) null); }
/** * Configure the output storage handler * @param jobContext the job context * @throws IOException */ @SuppressWarnings("unchecked") static void configureOutputStorageHandler( JobContext jobContext) throws IOException { configureOutputStorageHandler(jobContext, (List<String>) null); }
/** * Configure the output storage handler * @param jobContext the job context * @throws IOException */ @SuppressWarnings("unchecked") static void configureOutputStorageHandler( JobContext jobContext) throws IOException { configureOutputStorageHandler(jobContext, (List<String>) null); }
/** * Configure the output storage handler, with allowing specification * of partvals from which it picks the dynamic partvals * @param context the job context * @param jobInfo the output job info * @param fullPartSpec * @throws IOException */ protected static void configureOutputStorageHandler( JobContext context, OutputJobInfo jobInfo, Map<String, String> fullPartSpec) throws IOException { List<String> dynamicPartKeys = jobInfo.getDynamicPartitioningKeys(); if ((dynamicPartKeys == null) || (dynamicPartKeys.isEmpty())) { configureOutputStorageHandler(context, (List<String>) null); } else { List<String> dynKeyVals = new ArrayList<String>(); for (String dynamicPartKey : dynamicPartKeys) { dynKeyVals.add(fullPartSpec.get(dynamicPartKey)); } configureOutputStorageHandler(context, dynKeyVals); } }
/** * Configure the output storage handler, with allowing specification * of partvals from which it picks the dynamic partvals * @param context the job context * @param jobInfo the output job info * @param fullPartSpec * @throws IOException */ protected static void configureOutputStorageHandler( JobContext context, OutputJobInfo jobInfo, Map<String, String> fullPartSpec) throws IOException { List<String> dynamicPartKeys = jobInfo.getDynamicPartitioningKeys(); if ((dynamicPartKeys == null) || (dynamicPartKeys.isEmpty())) { configureOutputStorageHandler(context, (List<String>) null); } else { List<String> dynKeyVals = new ArrayList<String>(); for (String dynamicPartKey : dynamicPartKeys) { dynKeyVals.add(fullPartSpec.get(dynamicPartKey)); } configureOutputStorageHandler(context, dynKeyVals); } }
/** * Configure the output storage handler, with allowing specification * of partvals from which it picks the dynamic partvals * @param context the job context * @param jobInfo the output job info * @param fullPartSpec * @throws IOException */ protected static void configureOutputStorageHandler( JobContext context, OutputJobInfo jobInfo, Map<String, String> fullPartSpec) throws IOException { List<String> dynamicPartKeys = jobInfo.getDynamicPartitioningKeys(); if ((dynamicPartKeys == null) || (dynamicPartKeys.isEmpty())) { configureOutputStorageHandler(context, (List<String>) null); } else { List<String> dynKeyVals = new ArrayList<String>(); for (String dynamicPartKey : dynamicPartKeys) { dynKeyVals.add(fullPartSpec.get(dynamicPartKey)); } configureOutputStorageHandler(context, dynKeyVals); } }
/** * Configure the output storage handler, with allowing specification * of partvals from which it picks the dynamic partvals * @param context the job context * @param jobInfo the output job info * @param fullPartSpec * @throws IOException */ protected static void configureOutputStorageHandler( JobContext context, OutputJobInfo jobInfo, Map<String, String> fullPartSpec) throws IOException { List<String> dynamicPartKeys = jobInfo.getDynamicPartitioningKeys(); if ((dynamicPartKeys == null) || (dynamicPartKeys.isEmpty())) { configureOutputStorageHandler(context, (List<String>) null); } else { List<String> dynKeyVals = new ArrayList<String>(); for (String dynamicPartKey : dynamicPartKeys) { dynKeyVals.add(fullPartSpec.get(dynamicPartKey)); } configureOutputStorageHandler(context, dynKeyVals); } }
/** * Gets the output format instance. * @param context the job context * @return the output format instance * @throws IOException */ protected OutputFormat<WritableComparable<?>, HCatRecord> getOutputFormat(JobContext context) throws IOException { OutputJobInfo jobInfo = getJobInfo(context.getConfiguration()); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo()); // Always configure storage handler with jobproperties/jobconf before calling any methods on it configureOutputStorageHandler(context); if (storageHandler instanceof FosterStorageHandler) { return new FileOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } else { return new DefaultOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } }
/** * Gets the output format instance. * @param context the job context * @return the output format instance * @throws IOException */ protected OutputFormat<WritableComparable<?>, HCatRecord> getOutputFormat(JobContext context) throws IOException { OutputJobInfo jobInfo = getJobInfo(context.getConfiguration()); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo()); // Always configure storage handler with jobproperties/jobconf before calling any methods on it configureOutputStorageHandler(context); if (storageHandler instanceof FosterStorageHandler) { return new FileOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } else { return new DefaultOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } }
/** * Gets the output format instance. * @param context the job context * @return the output format instance * @throws IOException */ protected OutputFormat<WritableComparable<?>, HCatRecord> getOutputFormat(JobContext context) throws IOException { OutputJobInfo jobInfo = getJobInfo(context.getConfiguration()); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo()); // Always configure storage handler with jobproperties/jobconf before calling any methods on it configureOutputStorageHandler(context); if (storageHandler instanceof FosterStorageHandler) { return new FileOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } else { return new DefaultOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } }
/** * Gets the output format instance. * @param context the job context * @return the output format instance * @throws IOException */ protected OutputFormat<WritableComparable<?>, HCatRecord> getOutputFormat(JobContext context) throws IOException { OutputJobInfo jobInfo = getJobInfo(context.getConfiguration()); HiveStorageHandler storageHandler = HCatUtil.getStorageHandler(context.getConfiguration(), jobInfo.getTableInfo().getStorerInfo()); // Always configure storage handler with jobproperties/jobconf before calling any methods on it configureOutputStorageHandler(context); if (storageHandler instanceof FosterStorageHandler) { return new FileOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } else { return new DefaultOutputFormatContainer(ReflectionUtils.newInstance( storageHandler.getOutputFormatClass(),context.getConfiguration())); } }