How to use

read

method

in

org.apache.beam.sdk.io.AvroIO

@Test
@Category(NeedsRunner.class)
public void testWriteThenReadGeneratedClassWithClass() throws Throwable {
 testWriteThenReadGeneratedClass(
   AvroIO.write(AvroGeneratedUser.class), AvroIO.read(AvroGeneratedUser.class));
}

PAssert.that(
    readPipeline.apply(
      "Read", AvroIO.read(GenericClass.class).from(outputFile.getAbsolutePath())))
  .containsInAnyOrder(values);
PAssert.that(
    readPipeline.apply(
      "Read withHintMatchesManyFiles",
      AvroIO.read(GenericClass.class)
        .from(outputFile.getAbsolutePath())
        .withHintMatchesManyFiles()))

@Test
public void testReadDisplayData() {
 AvroIO.Read<String> read = AvroIO.read(String.class).from("/foo.*");
 DisplayData displayData = DisplayData.from(read);
 assertThat(displayData, hasDisplayItem("filePattern", "/foo.*"));
}

readPipeline.apply(
  "Read",
  AvroIO.read(GenericClass.class)
    .from(tmpFolder.getRoot().getAbsolutePath() + "/first*")
    .watchForNewFiles(

public static PipelineResult run(Options options) {
 Pipeline pipeline = Pipeline.create(options);
 BigtableIO.Write write =
   BigtableIO.write()
     .withProjectId(options.getBigtableProjectId())
     .withInstanceId(options.getBigtableInstanceId())
     .withTableId(options.getBigtableTableId());
 pipeline
   .apply("Read from Avro", AvroIO.read(BigtableRow.class).from(options.getInputFilePattern()))
   .apply("Transform to Bigtable", MapElements.via(new AvroToBigtableFn()))
   .apply("Write to Bigtable", write);
 return pipeline.run();
}

@Test
public void testAvroIOGetName() {
 assertEquals("AvroIO.Read", AvroIO.read(String.class).from("/tmp/foo*/baz").getName());
 assertEquals("AvroIO.Write", AvroIO.write(String.class).to("/tmp/foo/baz").getName());
}

@Test
@SuppressWarnings("unchecked")
@Category(NeedsRunner.class)
public void testCompressedWriteAndReadASingleFile() throws Throwable {
 List<GenericClass> values =
   ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
 File outputFile = tmpFolder.newFile("output.avro");
 writePipeline
   .apply(Create.of(values))
   .apply(
     AvroIO.write(GenericClass.class)
       .to(outputFile.getAbsolutePath())
       .withoutSharding()
       .withCodec(CodecFactory.deflateCodec(9)));
 writePipeline.run();
 PAssert.that(
     readPipeline.apply(AvroIO.read(GenericClass.class).from(outputFile.getAbsolutePath())))
   .containsInAnyOrder(values);
 readPipeline.run();
 try (DataFileStream dataFileStream =
   new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader())) {
  assertEquals("deflate", dataFileStream.getMetaString("avro.codec"));
 }
}

@Test
@SuppressWarnings("unchecked")
@Category(NeedsRunner.class)
public void testWriteThenReadASingleFileWithNullCodec() throws Throwable {
 List<GenericClass> values =
   ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
 File outputFile = tmpFolder.newFile("output.avro");
 writePipeline
   .apply(Create.of(values))
   .apply(
     AvroIO.write(GenericClass.class)
       .to(outputFile.getAbsolutePath())
       .withoutSharding()
       .withCodec(CodecFactory.nullCodec()));
 writePipeline.run();
 PAssert.that(
     readPipeline.apply(AvroIO.read(GenericClass.class).from(outputFile.getAbsolutePath())))
   .containsInAnyOrder(values);
 readPipeline.run();
 try (DataFileStream dataFileStream =
   new DataFileStream(new FileInputStream(outputFile), new GenericDatumReader())) {
  assertEquals("null", dataFileStream.getMetaString("avro.codec"));
 }
}

@Test
@Category(NeedsRunner.class)
public void testWriteThenReadJavaClass() throws Throwable {
 List<GenericClass> values =
   ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
 File outputFile = tmpFolder.newFile("output.avro");
 writePipeline
   .apply(Create.of(values))
   .apply(
     AvroIO.write(GenericClass.class)
       .to(writePipeline.newProvider(outputFile.getAbsolutePath()))
       .withoutSharding());
 writePipeline.run();
 PAssert.that(
     readPipeline.apply(
       "Read",
       AvroIO.read(GenericClass.class)
         .from(readPipeline.newProvider(outputFile.getAbsolutePath()))))
   .containsInAnyOrder(values);
 readPipeline.run();
}

.apply(
  "Read",
  AvroIO.read(GenericClass.class)
    .from(readPipeline.newProvider(outputFile.getAbsolutePath())))
.apply(

/**
 * Tests that {@code AvroIO} can read an upgraded version of an old class, as long as the schema
 * resolution process succeeds. This test covers the case when a new, {@code @Nullable} field has
 * been added.
 *
 * <p>For more information, see http://avro.apache.org/docs/1.7.7/spec.html#Schema+Resolution
 */
@Test
@Category(NeedsRunner.class)
public void testWriteThenReadSchemaUpgrade() throws Throwable {
 List<GenericClass> values =
   ImmutableList.of(new GenericClass(3, "hi"), new GenericClass(5, "bar"));
 File outputFile = tmpFolder.newFile("output.avro");
 writePipeline
   .apply(Create.of(values))
   .apply(AvroIO.write(GenericClass.class).to(outputFile.getAbsolutePath()).withoutSharding());
 writePipeline.run();
 List<GenericClassV2> expected =
   ImmutableList.of(new GenericClassV2(3, "hi", null), new GenericClassV2(5, "bar", null));
 PAssert.that(
     readPipeline.apply(
       AvroIO.read(GenericClassV2.class).from(outputFile.getAbsolutePath())))
   .containsInAnyOrder(expected);
 readPipeline.run();
}