@Test public void testGetCodecFactoryIgnoresCase() { CodecFactory codecFactory = WriterUtils.getCodecFactory(Optional.of("SNAPPY"), Optional.<String>absent()); Assert.assertEquals(codecFactory.toString(), "snappy"); codecFactory = WriterUtils.getCodecFactory(Optional.of("snappy"), Optional.<String>absent()); Assert.assertEquals(codecFactory.toString(), "snappy"); } }
RandomData data1 = new RandomData(SCHEMA, COUNT, SEED); RandomData data2 = new RandomData(SCHEMA, COUNT, SEED+1); File file1 = makeFile((codec == null ? "null" : codec.toString()) + "-A"); File file2 = makeFile((codec2 == null ? "null" : codec2.toString()) + "-B"); DataFileWriter<Object> writer = new DataFileWriter<>(new GenericDatumWriter<>())
private static String codecToDescription (CodecFactory codec) { return codec.toString(); }
@Override public void writeExternal(ObjectOutput out) throws IOException { out.writeUTF(codecFactory.toString()); }
@Override public String toString() { checkNotNull(codecFactory, "Inner CodecFactory is null, please use non default constructor"); return codecFactory.toString(); } }
@Override public void writeExternal(ObjectOutput out) throws IOException { out.writeUTF(codecFactory.toString()); }
private boolean checkIsSupportedCodec(CodecFactory codecFactory) { final String codecStr = codecFactory.toString(); return noOptAvroCodecs.contains(codecStr) || deflatePattern.matcher(codecStr).matches() || xzPattern.matcher(codecStr).matches(); }
private boolean checkIsSupportedCodec(CodecFactory codecFactory) { final String codecStr = codecFactory.toString(); return noOptAvroCodecs.contains(codecStr) || deflatePattern.matcher(codecStr).matches() || xzPattern.matcher(codecStr).matches(); }
public SerializableAvroCodecFactory(CodecFactory codecFactory) { checkNotNull(codecFactory, "Codec can't be null"); checkState(checkIsSupportedCodec(codecFactory), "%s is not supported", codecFactory.toString()); this.codecFactory = codecFactory; }
@Override public String toString() { checkNotNull(codecFactory, "Inner CodecFactory is null, please use non default constructor"); return codecFactory.toString(); } }
@Test public void testDefaultCodecsIn() throws Exception { for (String codec : avroCodecs) { SerializableAvroCodecFactory codecFactory = new SerializableAvroCodecFactory(CodecFactory.fromString(codec)); assertEquals((CodecFactory.fromString(codec).toString()), codecFactory.getCodec().toString()); } }
@Test public void testDefaultCodecsSerDe() throws Exception { for (String codec : avroCodecs) { SerializableAvroCodecFactory codecFactory = new SerializableAvroCodecFactory(CodecFactory.fromString(codec)); SerializableAvroCodecFactory serdeC = SerializableUtils.clone(codecFactory); assertEquals(CodecFactory.fromString(codec).toString(), serdeC.getCodec().toString()); } }
@Override public void populateDisplayData(DisplayData.Builder builder) { filenamePolicy.populateDisplayData(builder); builder.add(DisplayData.item("schema", schema.get().toString()).withLabel("Record Schema")); builder.addIfNotDefault( DisplayData.item("codec", codec.getCodec().toString()).withLabel("Avro Compression Codec"), AvroIO.TypedWrite.DEFAULT_SERIALIZABLE_CODEC.toString()); builder.include("Metadata", new Metadata()); } }
@Test public void testDeflateCodecSerDeWithLevels() throws Exception { for (int i = 0; i < 10; ++i) { SerializableAvroCodecFactory codecFactory = new SerializableAvroCodecFactory(CodecFactory.deflateCodec(i)); SerializableAvroCodecFactory serdeC = SerializableUtils.clone(codecFactory); assertEquals(CodecFactory.deflateCodec(i).toString(), serdeC.getCodec().toString()); } }
@Test public void testXZCodecSerDeWithLevels() throws Exception { for (int i = 0; i < 10; ++i) { SerializableAvroCodecFactory codecFactory = new SerializableAvroCodecFactory(CodecFactory.xzCodec(i)); SerializableAvroCodecFactory serdeC = SerializableUtils.clone(codecFactory); assertEquals(CodecFactory.xzCodec(i).toString(), serdeC.getCodec().toString()); } }
@Test @SuppressWarnings("unchecked") public void testWriteWithSerDeCustomXZCodec() throws Exception { AvroIO.Write<String> write = AvroIO.write(String.class).to("/tmp/foo/baz").withCodec(CodecFactory.xzCodec(9)); assertEquals( CodecFactory.xzCodec(9).toString(), SerializableUtils.clone(write.inner.getCodec()).getCodec().toString()); }
@Test @SuppressWarnings("unchecked") public void testWriteWithSerDeCustomDeflateCodec() throws Exception { AvroIO.Write<String> write = AvroIO.write(String.class).to("/tmp/foo/baz").withCodec(CodecFactory.deflateCodec(9)); assertEquals( CodecFactory.deflateCodec(9).toString(), SerializableUtils.clone(write.inner.getCodec()).getCodec().toString()); }
@Test public void testWriteWithDefaultCodec() throws Exception { AvroIO.Write<String> write = AvroIO.write(String.class).to("/tmp/foo/baz"); assertEquals(CodecFactory.snappyCodec().toString(), write.inner.getCodec().toString()); }
@Test public void testWriteDisplayData() { AvroIO.Write<GenericClass> write = AvroIO.write(GenericClass.class) .to("/foo") .withShardNameTemplate("-SS-of-NN-") .withSuffix("bar") .withNumShards(100) .withCodec(CodecFactory.deflateCodec(6)); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("filePrefix", "/foo")); assertThat(displayData, hasDisplayItem("shardNameTemplate", "-SS-of-NN-")); assertThat(displayData, hasDisplayItem("fileSuffix", "bar")); assertThat( displayData, hasDisplayItem( "schema", "{\"type\":\"record\",\"name\":\"GenericClass\",\"namespace\":\"org.apache.beam.sdk.io" + ".AvroIOTest$\",\"fields\":[{\"name\":\"intField\",\"type\":\"int\"}," + "{\"name\":\"stringField\",\"type\":\"string\"}]}")); assertThat(displayData, hasDisplayItem("numShards", 100)); assertThat(displayData, hasDisplayItem("codec", CodecFactory.deflateCodec(6).toString())); } }