How to use

withType

method

in

org.kitesdk.data.mapreduce.DatasetKeyInputFormat$ConfigBuilder

@SuppressWarnings("deprecation")
private Job createJob() throws Exception {
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 DatasetKeyOutputFormat.configure(job).writeTo(outputDataset).withType(GenericData.Record.class);
 return job;
}

@Test
@SuppressWarnings("deprecation")
public void testJobAppend() throws Exception {
 populateInputDataset();
 populateOutputDataset(); // existing output will be overwritten
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 DatasetKeyOutputFormat.configure(job).appendTo(outputDataset).withType(GenericData.Record.class);
 Assert.assertTrue(job.waitForCompletion(true));
 checkOutput(true);
}

@Test
@SuppressWarnings("deprecation")
public void testJobOverwrite() throws Exception {
 populateInputDataset();
 populateOutputDataset(); // existing output will be overwritten
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 DatasetKeyOutputFormat.configure(job).overwrite(outputDataset).withType(GenericData.Record.class);
 Assert.assertTrue(job.waitForCompletion(true));
 checkOutput(false);
}

public void run() throws IOException {
 Configuration conf = new Configuration();
 DatasetKeyInputFormat.configure(conf).readFrom(eventsUri).withType(StandardEvent.class);
 DatasetKeyOutputFormat.configure(conf).writeTo(correlatedEventsUri).withType(CorrelatedEvents.class);

/**
 * Adds configuration for {@code DatasetKeyInputFormat} to read from the
 * given {@link Dataset} or {@link View} instance.
 *
 * @param view a dataset or view
 * @return this for method chaining
 */
public ConfigBuilder readFrom(View<?> view) {
 DatasetDescriptor descriptor = view.getDataset().getDescriptor();
 // if this is a partitioned dataset, add the partition location
 if (view instanceof FileSystemDataset) {
  conf.set(KITE_PARTITION_DIR, String.valueOf(descriptor.getLocation()));
 }
 // add descriptor properties to the config
 for (String property : descriptor.listProperties()) {
  conf.set(property, descriptor.getProperty(property));
 }
 if (DataModelUtil.isGeneric(view.getType())) {
  Schema datasetSchema = view.getDataset().getDescriptor().getSchema();
  // only set the read schema if the view is a projection
  if (!datasetSchema.equals(view.getSchema())) {
   withSchema(view.getSchema());
  }
 } else {
  withType(view.getType());
 }
 conf.set(KITE_INPUT_URI, view.getUri().toString());
 return this;
}

/**
 * Adds configuration for {@code DatasetKeyInputFormat} to read from the
 * given {@link Dataset} or {@link View} instance.
 *
 * @param view a dataset or view
 * @return this for method chaining
 */
public ConfigBuilder readFrom(View<?> view) {
 DatasetDescriptor descriptor = view.getDataset().getDescriptor();
 // if this is a partitioned dataset, add the partition location
 if (view instanceof FileSystemDataset) {
  conf.set(KITE_PARTITION_DIR, String.valueOf(descriptor.getLocation()));
 }
 // add descriptor properties to the config
 for (String property : descriptor.listProperties()) {
  conf.set(property, descriptor.getProperty(property));
 }
 if (DataModelUtil.isGeneric(view.getType())) {
  Schema datasetSchema = view.getDataset().getDescriptor().getSchema();
  // only set the read schema if the view is a projection
  if (!datasetSchema.equals(view.getSchema())) {
   withSchema(view.getSchema());
  }
 } else {
  withType(view.getType());
 }
 conf.set(KITE_INPUT_URI, view.getUri().toString());
 return this;
}

@Test
@SuppressWarnings("deprecation")
public void testSignalReadyOutputView() throws Exception {
 Assume.assumeTrue(!Hadoop.isHadoop1());
 populateInputDataset();
 populateOutputDataset(); // existing output will be overwritten
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 View<Record> outputView = outputDataset.with("name", "apple", "banana", "carrot");
 DatasetKeyOutputFormat.configure(job).appendTo(outputView).withType(GenericData.Record.class);
 Assert.assertTrue(job.waitForCompletion(true));
 Assert.assertFalse("Output dataset should not be signaled ready",
   ((Signalable)outputDataset).isReady());
 Assert.assertTrue("Output view should be signaled ready",
   ((Signalable)outputView).isReady());
}

@Test
@SuppressWarnings("deprecation")
public void testJobOutputDatasetSignaledReady() throws Exception {
 Assume.assumeTrue(!Hadoop.isHadoop1());
 populateInputDataset();
 populateOutputDataset(); // existing output will be overwritten
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 DatasetKeyOutputFormat.configure(job).overwrite(outputDataset).withType(GenericData.Record.class);
 Assert.assertTrue(job.waitForCompletion(true));
 Assert.assertTrue("Output dataset should be signaled ready",
   ((Signalable)outputDataset).isReady());
}