private static void configureAvroShuffle(JobConf job) { job.setOutputKeyComparatorClass(AvroKeyComparator.class); job.setMapOutputKeyClass(AvroKey.class); job.setMapOutputValueClass(AvroValue.class); // add AvroSerialization to io.serializations Collection<String> serializations = job.getStringCollection("io.serializations"); if (!serializations.contains(AvroSerialization.class.getName())) { serializations.add(AvroSerialization.class.getName()); job.setStrings("io.serializations", serializations.toArray(new String[0])); } }
Class<? extends InputFormat> inputFormat) { job.setInputFormat(inputFormat); job.setMapOutputValueClass(outputValueClass); job.setMapOutputKeyClass(outputKeyClass); job.setMapperClass(mapper); job.setStrings("io.serializations", job.get("io.serializations"), MutationSerialization.class.getName(), ResultSerialization.class.getName());
protected JobConf configSaxpyTextoutput (Path py, Path px, Path saxpy_output, double a) throws Exception { final JobConf conf = new JobConf(getConf(), SaxpyTextoutput.class); conf.set("y_path", py.getName()); conf.set("x_path", px.getName()); conf.set("a", "" + a); conf.setJobName("SaxpyTextoutput"); conf.setMapperClass(SaxpyTextoutput.MapStage1.class); conf.setReducerClass(SaxpyTextoutput.RedStage1.class); FileInputFormat.setInputPaths(conf, py, px); FileOutputFormat.setOutputPath(conf, saxpy_output); conf.setNumReduceTasks( nreducers ); conf.setOutputKeyClass(IntWritable.class); conf.setMapOutputValueClass(DoubleWritable.class); conf.setOutputValueClass(Text.class); return conf; } }
public static void runJob(String src, String dest) throws Exception { JobConf job = new JobConf(); job.setJarByClass(HttpDownloadMap.class); FileSystem fs = FileSystem.get(job); Path destination = new Path(dest); fs.delete(destination, true); job.setMapperClass(HttpDownloadMap.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); FileInputFormat.setInputPaths(job, src); FileOutputFormat.setOutputPath(job, destination); JobClient.runJob(job); } }
MultiTableSnapshotInputFormat.setInput(job, snapshotScans, tmpRestoreDir); job.setInputFormat(MultiTableSnapshotInputFormat.class); if (outputValueClass != null) { job.setMapOutputValueClass(outputValueClass); job.setMapOutputKeyClass(outputKeyClass); job.setMapperClass(mapper); if (addDependencyJars) { addDependencyJars(job);
protected JobConf configStage1 (String out_prefix) throws Exception { final JobConf conf = new JobConf(getConf(), MatvecPrep.class); conf.set("block_size", "" + block_size); conf.set("matrix_row", "" + number_nodes); conf.set("out_prefix", "" + out_prefix); conf.set("makesym", "" + makesym); conf.setJobName("MatvecPrep_Stage1"); conf.setMapperClass(MapStage1.class); conf.setReducerClass(RedStage1.class); FileSystem fs = FileSystem.get(getConf()); fs.delete(output_path, true); FileInputFormat.setInputPaths(conf, edge_path); FileOutputFormat.setOutputPath(conf, output_path); int num_reduce_tasks = nreducer; conf.setNumReduceTasks( num_reduce_tasks ); conf.setOutputKeyClass(Text.class); conf.setOutputValueClass(Text.class); conf.setMapOutputValueClass(Text.class); return conf; } }
JobConf cfg = new JobConf(); cfg.setMapOutputKeyClass(IntWritable.class); cfg.setMapOutputValueClass(Text.class); cfg.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, CustomSerialization.class.getName());
private static void setupTetherJob(JobConf job) throws IOException { job.setMapRunnerClass(TetherMapRunner.class); job.setPartitionerClass(TetherPartitioner.class); job.setReducerClass(TetherReducer.class); job.setInputFormat(TetherInputFormat.class); job.setOutputFormat(TetherOutputFormat.class); job.setOutputKeyClass(TetherData.class); job.setOutputKeyComparatorClass(TetherKeyComparator.class); job.setMapOutputValueClass(NullWritable.class); // set the map output key class to TetherData job.setMapOutputKeyClass(TetherData.class); // if protocol isn't set if (job.getStrings(TETHER_PROTOCOL)==null) { job.set(TETHER_PROTOCOL, "sasl"); } // add TetherKeySerialization to io.serializations Collection<String> serializations = job.getStringCollection("io.serializations"); if (!serializations.contains(TetherKeySerialization.class.getName())) { serializations.add(TetherKeySerialization.class.getName()); job.setStrings("io.serializations", serializations.toArray(new String[0])); } // determine whether the executable should be added to the cache. if (job.getBoolean(TETHER_EXEC_CACHED,false)){ DistributedCache.addCacheFile(getExecutable(job), job); } }
protected JobConf configPass2 () throws Exception { final JobConf conf = new JobConf(getConf(), MatvecNaive.class); conf.set("number_nodes", "" + number_nodes); conf.setJobName("MatvecNaive_pass2"); conf.setMapperClass(MapPass2.class); conf.setReducerClass(RedPass2.class); FileInputFormat.setInputPaths(conf, tempmv_path); FileOutputFormat.setOutputPath(conf, output_path); conf.setNumReduceTasks( nreducer ); conf.setOutputKeyClass(LongWritable.class); conf.setMapOutputValueClass(DoubleWritable.class); conf.setOutputValueClass(Text.class); return conf; }
JobConf conf = new JobConf(MyJob.class); ... conf.setMapOutputKeyClass(K2.class); conf.setMapOutputValueClass(V2.class);
if(!isAvro) { conf.setPartitionerClass(HadoopStoreBuilderPartitioner.class); conf.setMapperClass(mapperClass); conf.setMapOutputKeyClass(BytesWritable.class); conf.setMapOutputValueClass(BytesWritable.class); conf.setReducerClass(HadoopStoreBuilderReducer.class); conf.setInputFormat(inputFormatClass); conf.setOutputFormat(SequenceFileOutputFormat.class); conf.setOutputKeyClass(BytesWritable.class); conf.setOutputValueClass(BytesWritable.class); conf.setPartitionerClass(AvroStoreBuilderPartitioner.class); conf.setMapOutputKeyClass(ByteBuffer.class); conf.setMapOutputValueClass(ByteBuffer.class); conf.setInputFormat(inputFormatClass); conf.setOutputFormat((Class<? extends OutputFormat>) AvroOutputFormat.class); conf.setOutputKeyClass(ByteBuffer.class); conf.setOutputValueClass(ByteBuffer.class); conf.setReducerClass(AvroStoreBuilderReducer.class);
protected JobConf configPass1 () throws Exception { final JobConf conf = new JobConf(getConf(), MatvecNaive.class); conf.set("number_nodes", "" + number_nodes); conf.set("makesym", "" + makesym); conf.set("transpose", "" + transpose); conf.set("ignore_weights", "" + ignore_weights); conf.setJobName("MatvecNaive_pass1"); conf.setMapperClass(MapPass1.class); conf.setReducerClass(RedPass1.class); if( vector_path == null ) FileInputFormat.setInputPaths(conf, edge_path); else FileInputFormat.setInputPaths(conf, edge_path, vector_path); FileOutputFormat.setOutputPath(conf, tempmv_path); conf.setNumReduceTasks( nreducer ); conf.setOutputKeyClass(LongWritable.class); conf.setOutputValueClass(DoubleWritable.class); conf.setMapOutputValueClass(Text.class); return conf; }
private static void configureAvroShuffle(JobConf job) { job.setOutputKeyComparatorClass(AvroKeyComparator.class); job.setMapOutputKeyClass(AvroKey.class); job.setMapOutputValueClass(AvroValue.class); // add AvroSerialization to io.serializations Collection<String> serializations = job.getStringCollection("io.serializations"); if (!serializations.contains(AvroSerialization.class.getName())) { serializations.add(AvroSerialization.class.getName()); job.setStrings("io.serializations", serializations.toArray(new String[0])); } }
success = true; HiveFileFormatUtils.prepareJobOutput(job); job.setOutputFormat(HiveOutputFormatImpl.class); job.setMapperClass(work.getMapperClass()); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(NullWritable.class); if(work.getNumMapTasks() != null) { job.setNumMapTasks(work.getNumMapTasks()); job.setInputFormat(JavaUtils.loadClass(inpFormat)); } catch (ClassNotFoundException e) { throw new RuntimeException(e.getMessage(), e);
protected JobConf configStage1() throws Exception { final JobConf conf = new JobConf(getConf(), PagerankInitVector.class); conf.set("number_nodes", "" + number_nodes); conf.setJobName("PagerankInitVector_Stage1"); conf.setMapperClass(MapStage1.class); conf.setReducerClass(RedStage1.class); FileInputFormat.setInputPaths(conf, initial_prinput_path); FileOutputFormat.setOutputPath(conf, output_path); conf.setNumReduceTasks( nreducers ); conf.setOutputKeyClass(IntWritable.class); conf.setMapOutputValueClass(Text.class); conf.setOutputValueClass(Text.class); return conf; } }
public void testKeyMismatch() throws Exception { configure(); // Set bad MapOutputKeyClass and MapOutputValueClass conf.setMapOutputKeyClass(IntWritable.class); conf.setMapOutputValueClass(IntWritable.class); RunningJob r_job = jc.submitJob(conf); while (!r_job.isComplete()) { Thread.sleep(1000); } if (r_job.isSuccessful()) { fail("Oops! The job was supposed to break due to an exception"); } }
job.setInputFormat(work.getInputformatClass()); job.setOutputFormat(HiveOutputFormatImpl.class); job.setMapperClass(MergeFileMapper.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(NullWritable.class); job.setOutputKeyClass(NullWritable.class); job.setOutputValueClass(NullWritable.class);
public void testValueMismatch() throws Exception { configure(); // Set good MapOutputKeyClass, bad MapOutputValueClass conf.setMapOutputKeyClass(Text.class); conf.setMapOutputValueClass(IntWritable.class); RunningJob r_job = jc.submitJob(conf); while (!r_job.isComplete()) { Thread.sleep(1000); } if (r_job.isSuccessful()) { fail("Oops! The job was supposed to break due to an exception"); } }
success = true; HiveFileFormatUtils.prepareJobOutput(job); job.setOutputFormat(HiveOutputFormatImpl.class); job.setMapperClass(work.getMapperClass()); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(NullWritable.class); if(work.getNumMapTasks() != null) { job.setNumMapTasks(work.getNumMapTasks()); job.setInputFormat(JavaUtils.loadClass(inpFormat)); } catch (ClassNotFoundException e) { throw new RuntimeException(e.getMessage(), e);
public void testNoMismatch() throws Exception{ configure(); // Set good MapOutputKeyClass and MapOutputValueClass conf.setMapOutputKeyClass(Text.class); conf.setMapOutputValueClass(Text.class); RunningJob r_job = jc.submitJob(conf); while (!r_job.isComplete()) { Thread.sleep(1000); } if (!r_job.isSuccessful()) { fail("Oops! The job broke due to an unexpected error"); } } }