/** * Runs a pipeline to import Avro files in GCS to a Cloud Bigtable table. * * @param args arguments to the pipeline */ public static void main(String[] args) { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); PipelineResult result = run(options); // Wait for pipeline to finish only if it is not constructing a template. if (options.as(DataflowPipelineOptions.class).getTemplateLocation() == null) { result.waitUntilFinish(); } }
/** * Runs a pipeline to export data from a Cloud Bigtable table to Avro files in GCS. * * @param args arguments to the pipeline */ public static void main(String[] args) { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); PipelineResult result = run(options); // Wait for pipeline to finish only if it is not constructing a template. if (options.as(DataflowPipelineOptions.class).getTemplateLocation() == null) { result.waitUntilFinish(); } }
|| !isNullOrEmpty(options.getTemplateLocation())) { boolean isTemplate = !isNullOrEmpty(options.getTemplateLocation()); if (isTemplate) { checkArgument( firstNonNull(options.getTemplateLocation(), options.getDataflowJobFile()); checkArgument( fileLocation.startsWith("/") || fileLocation.startsWith("gs://"),
public static void main(String[] args) { Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class); Pipeline p = Pipeline.create(options); SpannerConfig spannerConfig = SpannerConfig.create() .withHost(options.getSpannerHost()) .withInstanceId(options.getInstanceId()) .withDatabaseId(options.getDatabaseId()); p.apply(new ImportTransform(spannerConfig, options.getInputDir(), options.getWaitForIndexes())); PipelineResult result = p.run(); if (options.getWaitUntilFinish() && /* Only if template location is null, there is a dataflow job to wait for. Else it's * template generation which doesn't start a dataflow job. */ options.as(DataflowPipelineOptions.class).getTemplateLocation() == null) { result.waitUntilFinish(); } } }
options.as(DataflowPipelineOptions.class).getTemplateLocation() == null) { result.waitUntilFinish();
public static PipelineResult run(Options options) { Pipeline pipeline = Pipeline.create(options); BigtableIO.Read read = BigtableIO.read() .withProjectId(options.getBigtableProjectId()) .withInstanceId(options.getBigtableInstanceId()) .withTableId(options.getBigtableTableId()); // Do not validate input fields if it is running as a template. if (options.as(DataflowPipelineOptions.class).getTemplateLocation() != null) { read = read.withoutValidation(); } ValueProvider<String> filePathPrefix = DualInputNestedValueProvider.of( options.getOutputDirectory(), options.getFilenamePrefix(), new SerializableFunction<TranslatorInput<String, String>, String>() { @Override public String apply(TranslatorInput<String, String> input) { return new StringBuilder(input.getX()).append(input.getY()).toString(); } }); pipeline .apply("Read from Bigtable", read) .apply("Transform to Avro", MapElements.via(new BigtableToAvroFn())) .apply( "Write to Avro in GCS", AvroIO.write(BigtableRow.class).to(filePathPrefix).withSuffix(".avro")); return pipeline.run(); }