@Override protected DataWriter<FileAwareInputStream> buildWriter() throws IOException { return new TarArchiveInputStreamDataWriter(this.destination.getProperties(), this.branches, this.branch); }
protected DataWriter<FileAwareInputStream> buildWriter() throws IOException { return new FileAwareInputStreamDataWriter(this.destination.getProperties(), this.branches, this.branch, this.writerAttemptId); }
@Override public DataWriter<QueryBasedHiveConversionEntity> build() throws IOException { try { return new HiveQueryExecutionWriter(HiveJdbcConnector.newConnectorWithProps(this.destination.getProperties().getProperties()), this.destination.getProperties()); } catch (SQLException e) { throw new RuntimeException(e); } } }
@Override public final DataWriter<FileAwareInputStream> build() throws IOException { setJobSpecificOutputPaths(this.destination.getProperties()); // Each writer/mapper gets its own task-staging directory this.destination.getProperties().setProp(ConfigurationKeys.WRITER_FILE_PATH, this.writerId); return buildWriter(); }
/** * Build a {@link org.apache.gobblin.writer.DataWriter}. * * @return the built {@link org.apache.gobblin.writer.DataWriter} * @throws java.io.IOException if there is anything wrong building the writer */ @Override public DataWriter<Object> build() throws IOException { return new MetadataWriterWrapper<byte[]>(new SimpleDataWriter(this, this.destination.getProperties()), byte[].class, this.branches, this.branch, this.destination.getProperties()); }
@Override public DataWriter<GenericRecord> build() throws IOException { Preconditions.checkNotNull(this.destination); Preconditions.checkArgument(!Strings.isNullOrEmpty(this.writerId)); Preconditions.checkNotNull(this.schema); Preconditions.checkArgument(this.format == WriterOutputFormat.AVRO); switch (this.destination.getType()) { case HDFS: return new AvroHdfsDataWriter(this, this.destination.getProperties()); default: throw new RuntimeException("Unknown destination type: " + this.destination.getType()); } } }
@Override public DataWriter<Group> build() throws IOException { Preconditions.checkNotNull(this.destination); Preconditions.checkArgument(!Strings.isNullOrEmpty(this.writerId)); Preconditions.checkNotNull(this.schema); Preconditions.checkArgument(this.format == WriterOutputFormat.PARQUET); switch (this.destination.getType()) { case HDFS: return new ParquetHdfsDataWriter(this, this.destination.getProperties()); default: throw new RuntimeException("Unknown destination type: " + this.destination.getType()); } }
/** * For backward compatibility on how Fork creates writer, invoke fromState when it's called writeTo method. * @param destination * @return this */ @Override public AsyncHttpWriterBuilder<D, RQ, RP> writeTo(Destination destination) { super.writeTo(destination); return fromState(destination.getProperties()); }
/** * @param destination * @return Provides JdbcWriterCommands bases on destination. */ public JdbcWriterCommands newInstance(Destination destination, Connection conn) { switch (destination.getType()) { case MYSQL: return new MySqlWriterCommands(destination.getProperties(), conn); case TERADATA: return new TeradataWriterCommands(destination.getProperties(), conn); case POSTGRES: return new PostgresWriterCommands(destination.getProperties(), conn); default: throw new IllegalArgumentException(destination.getType() + " is not supported"); } }
@Override protected List<StreamCodec> buildEncoders() { Preconditions.checkNotNull(this.destination, "Destination must be set before building encoders"); List<StreamCodec> encoders = new ArrayList<>(); // TODO: refactor this when capability support comes back in // Compress first since compressing encrypted data will give no benefit Map<String, Object> compressionConfig = CompressionConfigParser.getConfigForBranch(this.destination.getProperties(), this.branches, this.branch); if (compressionConfig != null) { encoders.add(CompressionFactory.buildStreamCompressor(compressionConfig)); } Map<String, Object> encryptionConfig = EncryptionConfigParser .getConfigForBranch(EncryptionConfigParser.EntityType.WRITER, this.destination.getProperties(), this.branches, this.branch); if (encryptionConfig != null) { encoders.add(EncryptionFactory.buildStreamCryptoProvider(encryptionConfig)); } return encoders; } }
private CompressionCodecName getCodecFromConfig() { State state = this.destination.getProperties(); String codecValue = Optional.ofNullable(state.getProp(getProperty(WRITER_CODEC_TYPE))) .orElse(CompressionCodecName.SNAPPY.toString()); return CompressionCodecName.valueOf(codecValue.toUpperCase()); }
/** * For backward compatibility on how Fork creates writer, invoke fromState when it's called writeTo method. * @param destination * @return */ @Override public B writeTo(Destination destination) { super.writeTo(destination); fromState(destination.getProperties()); return typedSelf(); }
private ParquetProperties.WriterVersion getWriterVersion() { return ParquetProperties.WriterVersion.fromString( this.destination.getProperties().getProp(getProperty(WRITER_PARQUET_VERSION), DEFAULT_PARQUET_WRITER)); }
@SuppressWarnings("deprecation") @Override public DataWriter<Writable> build() throws IOException { Preconditions.checkNotNull(this.destination); Preconditions.checkArgument(!Strings.isNullOrEmpty(this.writerId)); State properties = this.destination.getProperties(); if (!properties.contains(WRITER_WRITABLE_CLASS) || !properties.contains(WRITER_OUTPUT_FORMAT_CLASS)) { HiveSerDeWrapper serializer = HiveSerDeWrapper.getSerializer(properties); properties.setProp(WRITER_WRITABLE_CLASS, serializer.getSerDe().getSerializedClass().getName()); properties.setProp(WRITER_OUTPUT_FORMAT_CLASS, serializer.getOutputFormatClassName()); } return new HiveWritableHdfsDataWriter(this, properties); }
@Test public void testMethods() { State state = new State(); state.setProp("foo", "bar"); Destination destination = Destination.of(Destination.DestinationType.HDFS, state); Assert.assertEquals(destination.getType(), Destination.DestinationType.HDFS); Assert.assertEquals(destination.getProperties().getPropertyNames().size(), 1); Assert.assertEquals(destination.getProperties().getProp("foo"), "bar"); } }
public Mode getDefaultMode() { try { State destinationCfg = getDestination().getProperties(); String modeKey = ForkOperatorUtils.getPathForBranch(destinationCfg, FULL_MODE_KEY, getBranches(), getBranch()); return Mode.valueOf(destinationCfg.getProp(modeKey, Mode.POST_RECORDS.name()).toUpperCase()); } catch (Throwable t) { return Mode.POST_RECORDS; } }
public JdbcWriter(JdbcWriterBuilder builder) { this.state = builder.destination.getProperties(); this.state.setProp(ConfigurationKeys.FORK_BRANCH_ID_KEY, Integer.toString(builder.branch)); String databaseTableKey = ForkOperatorUtils.getPropertyNameForBranch(JdbcPublisher.JDBC_PUBLISHER_DATABASE_NAME, builder.branches, builder.branch); this.databaseName = Preconditions.checkNotNull(this.state.getProp(databaseTableKey), "Staging table is missing with key " + databaseTableKey); String stagingTableKey = ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_STAGING_TABLE, builder.branches, builder.branch); this.tableName = Preconditions.checkNotNull(this.state.getProp(stagingTableKey), "Staging table is missing with key " + stagingTableKey); try { this.conn = createConnection(); this.commands = new JdbcWriterCommandsFactory().newInstance(this.state, this.conn); this.commands.setConnectionParameters(this.state.getProperties(), this.conn); } catch (SQLException e) { throw new RuntimeException(e); } }
public String getDefaultEventBusId() { State destinationCfg = getDestination().getProperties(); String eventBusIdKey = ForkOperatorUtils.getPathForBranch(destinationCfg, FULL_EVENTBUSID_KEY, getBranches(), getBranch()); if (destinationCfg.contains(eventBusIdKey)) { return destinationCfg.getProp(eventBusIdKey); } else { return WriterUtils.getWriterOutputDir(destinationCfg, getBranches(), getBranch()) .toString(); } }
@Override public DataWriter build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); CouchbaseEnvironment couchbaseEnvironment = CouchbaseEnvironmentFactory.getInstance(config); //TODO: Read config to decide whether to build a blocking writer or an async writer double failureAllowance = ConfigUtils.getDouble(config, CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_CONFIG, CouchbaseWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; boolean retriesEnabled = ConfigUtils.getBoolean(config, CouchbaseWriterConfigurationKeys.RETRIES_ENABLED, CouchbaseWriterConfigurationKeys.RETRIES_ENABLED_DEFAULT); int maxRetries = ConfigUtils.getInt(config, CouchbaseWriterConfigurationKeys.MAX_RETRIES, CouchbaseWriterConfigurationKeys.MAX_RETRIES_DEFAULT); // build an async couchbase writer AsyncDataWriter couchbaseWriter = new CouchbaseWriter(couchbaseEnvironment, config); return AsyncWriterManager.builder() .asyncDataWriter(couchbaseWriter) .failureAllowanceRatio(failureAllowance) .retriesEnabled(retriesEnabled) .numRetries(maxRetries) .config(config) .build(); } }
@Override public DataWriter build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); long commitTimeoutMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_CONFIG, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_DEFAULT); long commitStepWaitTimeMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_CONFIG, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_DEFAULT); double failureAllowance = ConfigUtils.getDouble(config, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_CONFIG, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; return AsyncWriterManager.builder() .config(config) .commitTimeoutMillis(commitTimeoutMillis) .commitStepWaitTimeInMillis(commitStepWaitTimeMillis) .failureAllowanceRatio(failureAllowance) .retriesEnabled(false) .asyncDataWriter(getAsyncDataWriter(taskProps)).maxOutstandingWrites(10000) .build(); } }