private JdbcWriterCommands createJdbcWriterCommands(Connection conn) { String destKey = ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY, this.branches, this.branchId); String destType = Preconditions.checkNotNull(this.state.getProp(destKey), destKey + " is required for underlying JDBC product name"); Destination dest = Destination.of(DestinationType.valueOf(destType.toUpperCase()), this.state); return this.jdbcWriterCommandsFactory.newInstance(dest, conn); }
private ParquetDataWriterBuilder getParquetDataWriterBuilder() { ParquetDataWriterBuilder writerBuilder = new ParquetDataWriterBuilder(); writerBuilder.destination = Destination.of(Destination.DestinationType.HDFS, properties); writerBuilder.writerId = TestConstants.TEST_WRITER_ID; writerBuilder.schema = this.schema; writerBuilder.format = WriterOutputFormat.PARQUET; return writerBuilder; }
/** * @param state * @return Provides JdbcWriterCommands based on ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY */ public JdbcWriterCommands newInstance(State state, Connection conn) { String destKey = ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY, state.getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1), state.getPropAsInt(ConfigurationKeys.FORK_BRANCH_ID_KEY, 0)); String destType = state.getProp(destKey); Preconditions.checkNotNull(destType, destKey + " is required for underlying JDBC product name"); return newInstance(Destination.of(DestinationType.valueOf(destType.toUpperCase()), state), conn); } }
.withWriterId("0").writeTo(Destination.of(Destination.DestinationType.HDFS, sourceState)) .writeInFormat(WriterOutputFormat.ORC).build();
/** * Build a {@link org.apache.gobblin.writer.DataWriter} for writing fetched data records. */ private DataWriter<Object> buildWriter() throws IOException { DataWriterBuilder<Object, Object> builder = this.taskContext.getDataWriterBuilder(this.branches, this.index) .writeTo(Destination.of(this.taskContext.getDestinationType(this.branches, this.index), this.taskState)) .writeInFormat(this.taskContext.getWriterOutputFormat(this.branches, this.index)).withWriterId(this.taskId) .withSchema(this.convertedSchema.orNull()).withBranches(this.branches).forBranch(this.index); if (this.taskAttemptId.isPresent()) { builder.withAttemptId(this.taskAttemptId.get()); } DataWriter<Object> writer = new PartitionedDataWriter<>(builder, this.taskContext.getTaskState()); logger.info("Wrapping writer " + writer); return new DataWriterWrapperBuilder<>(writer, this.taskState).build(); }
@Test public void testMethods() { State state = new State(); state.setProp("foo", "bar"); Destination destination = Destination.of(Destination.DestinationType.HDFS, state); Assert.assertEquals(destination.getType(), Destination.DestinationType.HDFS); Assert.assertEquals(destination.getProperties().getPropertyNames().size(), 1); Assert.assertEquals(destination.getProperties().getProp("foo"), "bar"); } }
@BeforeClass public void setUp() throws Exception { // Making the staging and/or output dirs if necessary File stagingDir = new File(TestConstants.TEST_STAGING_DIR); File outputDir = new File(TestConstants.TEST_OUTPUT_DIR); if (!stagingDir.exists()) { stagingDir.mkdirs(); } if (!outputDir.exists()) { outputDir.mkdirs(); } this.schema = new Schema.Parser().parse(TestConstants.AVRO_SCHEMA); this.filePath = TestConstants.TEST_EXTRACT_NAMESPACE.replaceAll("\\.", "/") + "/" + TestConstants.TEST_EXTRACT_TABLE + "/" + TestConstants.TEST_EXTRACT_ID + "_" + TestConstants.TEST_EXTRACT_PULL_TYPE; properties = new State(); properties.setProp(ConfigurationKeys.WRITER_BUFFER_SIZE, ConfigurationKeys.DEFAULT_BUFFER_SIZE); properties.setProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, TestConstants.TEST_FS_URI); properties.setProp(ConfigurationKeys.WRITER_STAGING_DIR, TestConstants.TEST_STAGING_DIR); properties.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, TestConstants.TEST_OUTPUT_DIR); properties.setProp(ConfigurationKeys.WRITER_FILE_PATH, this.filePath); properties.setProp(ConfigurationKeys.WRITER_FILE_NAME, TestConstants.TEST_FILE_NAME); // Build a writer to write test records this.writer = new AvroDataWriterBuilder().writeTo(Destination.of(Destination.DestinationType.HDFS, properties)) .writeInFormat(WriterOutputFormat.AVRO).withWriterId(TestConstants.TEST_WRITER_ID).withSchema(this.schema) .withBranches(1).forBranch(0).build(); }
.writeTo(Destination.of(Destination.DestinationType.HDFS, properties)).writeInFormat(WriterOutputFormat.AVRO) .withWriterId(WRITER_ID).withSchema(this.schema).withBranches(1).forBranch(0); this.writer = new PartitionedDataWriter<Schema, GenericRecord>(builder, properties);
writer = closer.register((HiveWritableHdfsDataWriter) new HiveWritableHdfsDataWriterBuilder<>().withBranches(1) .withWriterId("0").writeTo(Destination.of(DestinationType.HDFS, sourceState)) .writeInFormat(WriterOutputFormat.ORC).build());
@Test public void testBuilder() throws IOException, InterruptedException, TimeoutException { final String eventBusId = "/GobblinTestEventBusWriterTest/testBuilder"; GobblinTestEventBusWriter.Builder writerBuilder = new GobblinTestEventBusWriter.Builder(); WorkUnit wu = WorkUnit.createEmpty(); wu.setProp(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId); writerBuilder.writeTo(Destination.of(DestinationType.HDFS, wu)); Assert.assertEquals(writerBuilder.getEventBusId(), eventBusId); try(TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId)) { GobblinTestEventBusWriter writer = writerBuilder.build(); writer.write("event1"); writer.write("event2"); asserter.assertNextValueEq("event1"); asserter.assertNextValueEq("event2"); Assert.assertEquals(writer.recordsWritten(), 2); } }
private SimpleDataWriter buildSimpleDataWriter() throws IOException { SimpleDataWriterBuilder b = (SimpleDataWriterBuilder)new SimpleDataWriterBuilder() .writeTo(Destination.of(Destination.DestinationType.HDFS, properties)).writeInFormat(WriterOutputFormat.AVRO) .withWriterId(TestConstants.TEST_WRITER_ID).withSchema(this.schema).forBranch(0); return new SimpleDataWriter(b, properties); }
"localhost:" + kafkaTestHelper.getKafkaServerPort()); state.setProp(KAFKA_TOPIC, topic); Destination destination = Destination.of(Destination.DestinationType.KAFKA, state); Kafka09JsonObjectWriterBuilder writerBuilder = new Kafka09JsonObjectWriterBuilder(); writerBuilder.writeTo(destination);
.writeTo(Destination.of(Destination.DestinationType.HDFS, state)) .writeInFormat(WriterOutputFormat.AVRO) .withWriterId("writer-1")
private JdbcWriterCommands createJdbcWriterCommands(Connection conn) { String destKey = ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY, this.branches, this.branchId); String destType = Preconditions.checkNotNull(this.state.getProp(destKey), destKey + " is required for underlying JDBC product name"); Destination dest = Destination.of(DestinationType.valueOf(destType.toUpperCase()), this.state); return this.jdbcWriterCommandsFactory.newInstance(dest, conn); }
/** * @param state * @return Provides JdbcWriterCommands based on ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY */ public JdbcWriterCommands newInstance(State state, Connection conn) { String destKey = ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY, state.getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1), state.getPropAsInt(ConfigurationKeys.FORK_BRANCH_ID_KEY, 0)); String destType = state.getProp(destKey); Preconditions.checkNotNull(destType, destKey + " is required for underlying JDBC product name"); return newInstance(Destination.of(DestinationType.valueOf(destType.toUpperCase()), state), conn); } }
/** * Build a {@link org.apache.gobblin.writer.DataWriter} for writing fetched data records. */ private DataWriter<Object> buildWriter() throws IOException { DataWriterBuilder<Object, Object> builder = this.taskContext.getDataWriterBuilder(this.branches, this.index) .writeTo(Destination.of(this.taskContext.getDestinationType(this.branches, this.index), this.taskState)) .writeInFormat(this.taskContext.getWriterOutputFormat(this.branches, this.index)).withWriterId(this.taskId) .withSchema(this.convertedSchema.orNull()).withBranches(this.branches).forBranch(this.index); if (this.taskAttemptId.isPresent()) { builder.withAttemptId(this.taskAttemptId.get()); } DataWriter<Object> writer = new PartitionedDataWriter<>(builder, this.taskContext.getTaskState()); logger.info("Wrapping writer " + writer); return new DataWriterWrapperBuilder<>(writer, this.taskState).build(); }