private SimpleDataWriter buildSimpleDataWriter() throws IOException { SimpleDataWriterBuilder b = (SimpleDataWriterBuilder)new SimpleDataWriterBuilder() .writeTo(Destination.of(Destination.DestinationType.HDFS, properties)).writeInFormat(WriterOutputFormat.AVRO) .withWriterId(TestConstants.TEST_WRITER_ID).withSchema(this.schema).forBranch(0); return new SimpleDataWriter(b, properties); }
/** * Build a {@link org.apache.gobblin.writer.DataWriter} for writing fetched data records. */ private DataWriter<Object> buildWriter() throws IOException { DataWriterBuilder<Object, Object> builder = this.taskContext.getDataWriterBuilder(this.branches, this.index) .writeTo(Destination.of(this.taskContext.getDestinationType(this.branches, this.index), this.taskState)) .writeInFormat(this.taskContext.getWriterOutputFormat(this.branches, this.index)).withWriterId(this.taskId) .withSchema(this.convertedSchema.orNull()).withBranches(this.branches).forBranch(this.index); if (this.taskAttemptId.isPresent()) { builder.withAttemptId(this.taskAttemptId.get()); } DataWriter<Object> writer = new PartitionedDataWriter<>(builder, this.taskContext.getTaskState()); logger.info("Wrapping writer " + writer); return new DataWriterWrapperBuilder<>(writer, this.taskState).build(); }
@Override public DataWriter<D> get() { try { return builder.withWriterId(PartitionedDataWriter.this.baseWriterId + "_" + PartitionedDataWriter.this.writerIdSuffix++).build(); } catch (IOException e) { throw new RuntimeException("Error creating writer", e); } } }, state);
.withWriterId("0").writeTo(Destination.of(Destination.DestinationType.HDFS, sourceState)) .writeInFormat(WriterOutputFormat.ORC).build();
@Override public DataWriterBuilder<S, D> withWriterId(String writerId) { super.withWriterId(writerId); return typedSelf(); }
this.baseWriterId = builder.getWriterId(); this.closer = Closer.create(); this.writerBuilder = builder; this.partitioner = Optional.of(WriterPartitioner.class.cast(ConstructorUtils .invokeConstructor(Class.forName(state.getProp(ConfigurationKeys.WRITER_PARTITIONER_CLASS)), state, builder.getBranches(), builder.getBranch()))); Preconditions .checkArgument(this.builder.get().validatePartitionSchema(this.partitioner.get().partitionSchema()), String
@Override public DataWriterBuilder<S, D> withBranches(int branches) { super.withBranches(branches); return typedSelf(); }
@Override public B writeInFormat(WriterOutputFormat format) { super.writeInFormat(format); return typedSelf(); }
@Override public DataWriterBuilder<S, D> forBranch(int branch) { super.forBranch(branch); return typedSelf(); }
@Override public DataWriterBuilder<S, D> withSchema(S schema) { super.withSchema(schema); return typedSelf(); }
@Override public B writeTo(Destination destination) { super.writeTo(destination); return typedSelf(); }
private DataWriter<D> createPartitionWriter(GenericRecord partition) throws IOException { if (!this.builder.isPresent()) { throw new IOException("Writer builder not found. This is an error in the code."); } DataWriter dataWriter = this.builder.get().forPartition(partition).withWriterId(this.baseWriterId + "_" + this.writerIdSuffix++) .build(); this.isSpeculativeAttemptSafe = this.isSpeculativeAttemptSafe && this.isDataWriterForPartitionSafe(dataWriter); this.isWatermarkCapable = this.isWatermarkCapable && this.isDataWriterWatermarkCapable(dataWriter); return dataWriter; }
writer = closer.register((HiveWritableHdfsDataWriter) new HiveWritableHdfsDataWriterBuilder<>().withBranches(1) .withWriterId("0").writeTo(Destination.of(DestinationType.HDFS, sourceState)) .writeInFormat(WriterOutputFormat.ORC).build());
@Override public DataWriterBuilder<S, D> withWriterId(String writerId) { super.withWriterId(writerId); return typedSelf(); }
this.baseWriterId = builder.getWriterId(); this.closer = Closer.create(); this.writerBuilder = builder; this.partitioner = Optional.of(WriterPartitioner.class.cast(ConstructorUtils .invokeConstructor(Class.forName(state.getProp(ConfigurationKeys.WRITER_PARTITIONER_CLASS)), state, builder.getBranches(), builder.getBranch()))); Preconditions .checkArgument(this.builder.get().validatePartitionSchema(this.partitioner.get().partitionSchema()), String
@Override public DataWriterBuilder<S, D> withBranches(int branches) { super.withBranches(branches); return typedSelf(); }
@Override public B writeInFormat(WriterOutputFormat format) { super.writeInFormat(format); return typedSelf(); }
@Override public DataWriterBuilder<S, D> forBranch(int branch) { super.forBranch(branch); return typedSelf(); }
@Override public DataWriterBuilder<S, D> withSchema(S schema) { super.withSchema(schema); return typedSelf(); }
@Override public B writeTo(Destination destination) { super.writeTo(destination); return typedSelf(); }