/** * Creates and initializes a JobConf object that can be used to execute * the DAG. The configuration object will contain configurations from mapred-site * overlaid with key/value pairs from the conf object. Finally it will also * contain some hive specific configurations that do not change from DAG to DAG. * * @param hiveConf Current conf for the execution * @return JobConf base configuration for job execution * @throws IOException */ public JobConf createConfiguration(HiveConf hiveConf) throws IOException { hiveConf.setBoolean("mapred.mapper.new-api", false); JobConf conf = new JobConf(new TezConfiguration(hiveConf)); conf.set("mapred.output.committer.class", NullOutputCommitter.class.getName()); conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false); conf.setBoolean("mapred.committer.job.task.cleanup.needed", false); conf.setClass("mapred.output.format.class", HiveOutputFormatImpl.class, OutputFormat.class); conf.set(MRJobConfig.OUTPUT_KEY_CLASS, HiveKey.class.getName()); conf.set(MRJobConfig.OUTPUT_VALUE_CLASS, BytesWritable.class.getName()); conf.set("mapred.partitioner.class", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPARTITIONER)); conf.set("tez.runtime.partitioner.class", MRPartitioner.class.getName()); // Removing job credential entry/ cannot be set on the tasks conf.unset("mapreduce.job.credentials.binary"); hiveConf.stripHiddenConfigurations(conf); return conf; }
/** * Creates and initializes a JobConf object that can be used to execute * the DAG. The configuration object will contain configurations from mapred-site * overlaid with key/value pairs from the hiveConf object. Finally it will also * contain some hive specific configurations that do not change from DAG to DAG. * * @param hiveConf Current hiveConf for the execution * @return JobConf base configuration for job execution * @throws IOException */ public JobConf createConfiguration(HiveConf hiveConf) throws IOException { hiveConf.setBoolean("mapred.mapper.new-api", false); JobConf conf = new JobConf(new TezConfiguration(hiveConf)); conf.set("mapred.output.committer.class", NullOutputCommitter.class.getName()); conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false); conf.setBoolean("mapred.committer.job.task.cleanup.needed", false); conf.setClass("mapred.output.format.class", HiveOutputFormatImpl.class, OutputFormat.class); conf.set(MRJobConfig.OUTPUT_KEY_CLASS, HiveKey.class.getName()); conf.set(MRJobConfig.OUTPUT_VALUE_CLASS, BytesWritable.class.getName()); conf.set("mapred.partitioner.class", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPARTITIONER)); conf.set("tez.runtime.partitioner.class", MRPartitioner.class.getName()); // Removing job credential entry/ cannot be set on the tasks conf.unset("mapreduce.job.credentials.binary"); hiveConf.stripHiddenConfigurations(conf); return conf; }
edgeManagerDescriptor.setUserPayload(cpConfig.toUserPayload(new TezConfiguration(conf))); UnorderedPartitionedKVEdgeConfig cpEdgeConf = UnorderedPartitionedKVEdgeConfig.newBuilder(keyClass, valClass,
final TezConfiguration tezConfig = new TezConfiguration(true); tezConfig.addResource(conf);
final TezConfiguration tezConfig = new TezConfiguration(conf);
v.setVertexManagerPlugin( VertexManagerPluginDescriptor.create(CartesianProductVertexManager.class.getName()) .setUserPayload(cpConfig.toUserPayload(new TezConfiguration(conf))));
@Override protected TezConfiguration newConfig( TezConfiguration defaultConfig ) { return defaultConfig == null ? new TezConfiguration() : new TezConfiguration( defaultConfig ); }
@Override protected TezConfiguration newConfig( TezConfiguration defaultConfig ) { return defaultConfig == null ? new TezConfiguration() : new TezConfiguration( defaultConfig ); }
@Override public TezConfiguration getConfigCopy() { return new TezConfiguration( configuration ); }
public static TezConfiguration createConfiguration( Map<Object, Object> properties ) { TezConfiguration conf = new TezConfiguration(); copyProperties( conf, properties ); return conf; }
@Override public TezConfiguration getConfigCopy() { return new TezConfiguration( getConfig() ); }
public TezConfiguration getConfiguration() { return new TezConfiguration( configuration ); }
@Override public TezConfiguration mergeMapIntoConfig( TezConfiguration defaultConfig, Map<String, String> map ) { return HadoopUtil.mergeConf( new TezConfiguration( defaultConfig ), map, true ); } }
@Override public TezConfiguration mergeMapIntoConfig( TezConfiguration defaultConfig, Map<String, String> map ) { return HadoopUtil.mergeConf( new TezConfiguration( defaultConfig ), map, true ); } }
@ProcessConfiguration @Override public TezConfiguration getConfig() { if( flowConf == null ) initConfig( null, new TezConfiguration() ); return flowConf; }
@ProcessConfiguration @Override public TezConfiguration getConfig() { if( flowConf == null ) initConfig( null, new TezConfiguration() ); return flowConf; }
protected void initConfig( Map<Object, Object> properties, TezConfiguration parentConfig ) { if( properties != null ) parentConfig = createConfig( properties, parentConfig ); if( parentConfig == null ) // this is ok, getJobConf will pass a default parent in return; flowConf = new TezConfiguration( parentConfig ); // prevent local values from being shared flowConf.set( "fs.http.impl", HttpFileSystem.class.getName() ); flowConf.set( "fs.https.impl", HttpFileSystem.class.getName() ); UserGroupInformation.setConfiguration( flowConf ); flowStagingPath = createStagingRoot(); }
protected void initConfig( Map<Object, Object> properties, TezConfiguration parentConfig ) { if( properties != null ) parentConfig = createConfig( properties, parentConfig ); if( parentConfig == null ) // this is ok, getJobConf will pass a default parent in return; flowConf = new TezConfiguration( parentConfig ); // prevent local values from being shared flowConf.set( "fs.http.impl", HttpFileSystem.class.getName() ); flowConf.set( "fs.https.impl", HttpFileSystem.class.getName() ); UserGroupInformation.setConfiguration( flowConf ); flowStagingPath = createStagingRoot(); }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { TezConfiguration jobConf = new TezConfiguration( getConfiguration() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); int nodeNum = jobConf.getInt( "cascading.flow.node.num", 0 ); String partname = String.format( "-%05d-%05d-", stepNum, nodeNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new Hadoop2TezFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }
@Override public TupleEntryCollector openTrapForWrite( Tap trap ) throws IOException { TezConfiguration jobConf = new TezConfiguration( getConfiguration() ); int stepNum = jobConf.getInt( "cascading.flow.step.num", 0 ); int nodeNum = jobConf.getInt( "cascading.flow.node.num", 0 ); String partname = String.format( "-%05d-%05d-", stepNum, nodeNum ); jobConf.set( "cascading.tapcollector.partname", "%s%spart" + partname + "%05d" ); return trap.openForWrite( new Hadoop2TezFlowProcess( this, jobConf ), null ); // do not honor sinkmode as this may be opened across tasks }