Tabnine Logo
DatasetKeyInputFormat$ConfigBuilder.withType
Code IndexAdd Tabnine to your IDE (free)

How to use
withType
method
in
org.kitesdk.data.mapreduce.DatasetKeyInputFormat$ConfigBuilder

Best Java code snippets using org.kitesdk.data.mapreduce.DatasetKeyInputFormat$ConfigBuilder.withType (Showing top 8 results out of 315)

origin: kite-sdk/kite

@SuppressWarnings("deprecation")
private Job createJob() throws Exception {
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 DatasetKeyOutputFormat.configure(job).writeTo(outputDataset).withType(GenericData.Record.class);
 return job;
}
origin: kite-sdk/kite

@Test
@SuppressWarnings("deprecation")
public void testJobAppend() throws Exception {
 populateInputDataset();
 populateOutputDataset(); // existing output will be overwritten
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 DatasetKeyOutputFormat.configure(job).appendTo(outputDataset).withType(GenericData.Record.class);
 Assert.assertTrue(job.waitForCompletion(true));
 checkOutput(true);
}
origin: kite-sdk/kite

@Test
@SuppressWarnings("deprecation")
public void testJobOverwrite() throws Exception {
 populateInputDataset();
 populateOutputDataset(); // existing output will be overwritten
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 DatasetKeyOutputFormat.configure(job).overwrite(outputDataset).withType(GenericData.Record.class);
 Assert.assertTrue(job.waitForCompletion(true));
 checkOutput(false);
}
origin: kite-sdk/kite-examples

public void run() throws IOException {
 Configuration conf = new Configuration();
 DatasetKeyInputFormat.configure(conf).readFrom(eventsUri).withType(StandardEvent.class);
 DatasetKeyOutputFormat.configure(conf).writeTo(correlatedEventsUri).withType(CorrelatedEvents.class);
origin: org.kitesdk/kite-data-mapreduce

/**
 * Adds configuration for {@code DatasetKeyInputFormat} to read from the
 * given {@link Dataset} or {@link View} instance.
 *
 * @param view a dataset or view
 * @return this for method chaining
 */
public ConfigBuilder readFrom(View<?> view) {
 DatasetDescriptor descriptor = view.getDataset().getDescriptor();
 // if this is a partitioned dataset, add the partition location
 if (view instanceof FileSystemDataset) {
  conf.set(KITE_PARTITION_DIR, String.valueOf(descriptor.getLocation()));
 }
 // add descriptor properties to the config
 for (String property : descriptor.listProperties()) {
  conf.set(property, descriptor.getProperty(property));
 }
 if (DataModelUtil.isGeneric(view.getType())) {
  Schema datasetSchema = view.getDataset().getDescriptor().getSchema();
  // only set the read schema if the view is a projection
  if (!datasetSchema.equals(view.getSchema())) {
   withSchema(view.getSchema());
  }
 } else {
  withType(view.getType());
 }
 conf.set(KITE_INPUT_URI, view.getUri().toString());
 return this;
}
origin: kite-sdk/kite

/**
 * Adds configuration for {@code DatasetKeyInputFormat} to read from the
 * given {@link Dataset} or {@link View} instance.
 *
 * @param view a dataset or view
 * @return this for method chaining
 */
public ConfigBuilder readFrom(View<?> view) {
 DatasetDescriptor descriptor = view.getDataset().getDescriptor();
 // if this is a partitioned dataset, add the partition location
 if (view instanceof FileSystemDataset) {
  conf.set(KITE_PARTITION_DIR, String.valueOf(descriptor.getLocation()));
 }
 // add descriptor properties to the config
 for (String property : descriptor.listProperties()) {
  conf.set(property, descriptor.getProperty(property));
 }
 if (DataModelUtil.isGeneric(view.getType())) {
  Schema datasetSchema = view.getDataset().getDescriptor().getSchema();
  // only set the read schema if the view is a projection
  if (!datasetSchema.equals(view.getSchema())) {
   withSchema(view.getSchema());
  }
 } else {
  withType(view.getType());
 }
 conf.set(KITE_INPUT_URI, view.getUri().toString());
 return this;
}
origin: kite-sdk/kite

@Test
@SuppressWarnings("deprecation")
public void testSignalReadyOutputView() throws Exception {
 Assume.assumeTrue(!Hadoop.isHadoop1());
 populateInputDataset();
 populateOutputDataset(); // existing output will be overwritten
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 View<Record> outputView = outputDataset.with("name", "apple", "banana", "carrot");
 DatasetKeyOutputFormat.configure(job).appendTo(outputView).withType(GenericData.Record.class);
 Assert.assertTrue(job.waitForCompletion(true));
 Assert.assertFalse("Output dataset should not be signaled ready",
   ((Signalable)outputDataset).isReady());
 Assert.assertTrue("Output view should be signaled ready",
   ((Signalable)outputView).isReady());
}
origin: kite-sdk/kite

@Test
@SuppressWarnings("deprecation")
public void testJobOutputDatasetSignaledReady() throws Exception {
 Assume.assumeTrue(!Hadoop.isHadoop1());
 populateInputDataset();
 populateOutputDataset(); // existing output will be overwritten
 Job job = new Job();
 DatasetKeyInputFormat.configure(job).readFrom(inputDataset).withType(GenericData.Record.class);
 job.setMapperClass(LineCountMapper.class);
 job.setMapOutputKeyClass(Text.class);
 job.setMapOutputValueClass(IntWritable.class);
 job.setReducerClass(GenericStatsReducer.class);
 DatasetKeyOutputFormat.configure(job).overwrite(outputDataset).withType(GenericData.Record.class);
 Assert.assertTrue(job.waitForCompletion(true));
 Assert.assertTrue("Output dataset should be signaled ready",
   ((Signalable)outputDataset).isReady());
}
org.kitesdk.data.mapreduceDatasetKeyInputFormat$ConfigBuilderwithType

Javadoc

Sets the entity Class that the input Dataset should produce.

This Class is used to configure the input Dataset. If this class cannot be found during job setup, the job will fail and throw a org.kitesdk.data.TypeNotFoundException.

If the type is set, then the type's schema is used for the expected schema and #withSchema(Schema) should not be called. This may, however, be used at the same time if the type is a generic record subclass.

Popular methods of DatasetKeyInputFormat$ConfigBuilder

  • readFrom
    Adds configuration for DatasetKeyInputFormat to read from the given Dataset or View instance.
  • <init>
  • withSchema
    Sets the expected schema to use when reading records from the Dataset. If this schema is set, #withT

Popular in Java

  • Making http requests using okhttp
  • setScale (BigDecimal)
  • getSystemService (Context)
  • addToBackStack (FragmentTransaction)
  • HttpServer (com.sun.net.httpserver)
    This class implements a simple HTTP server. A HttpServer is bound to an IP address and port number a
  • FlowLayout (java.awt)
    A flow layout arranges components in a left-to-right flow, much like lines of text in a paragraph. F
  • GridBagLayout (java.awt)
    The GridBagLayout class is a flexible layout manager that aligns components vertically and horizonta
  • IOException (java.io)
    Signals a general, I/O-related error. Error details may be specified when calling the constructor, a
  • PrintWriter (java.io)
    Wraps either an existing OutputStream or an existing Writerand provides convenience methods for prin
  • Locale (java.util)
    Locale represents a language/country/variant combination. Locales are used to alter the presentatio
  • Github Copilot alternatives
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now