public Boolean getUseClientSink() { return this.getConf().getBooleanProperty(USE_CLIENT_SINK, false); }
public CassandraSinkConfiguration(@NonNull final Configuration conf) { this.conf = conf; ConfigUtil.checkMandatoryProperties(this.conf, this.getMandatoryProperties()); this.partitionKeys = this.splitString(this.conf.getProperty(PARTITION_KEYS).get()); if (this.conf.getProperty(INITIAL_HOSTS).isPresent()) { this.initialHosts = this.splitString(this.conf.getProperty(INITIAL_HOSTS).get()); } else { this.initialHosts = new ArrayList<>(); } // Source fields can be case insensitive so we convert everything to lower case for comparing this.filteredColumns = this.conf.getProperty(COLUMN_LIST).isPresent() ? Optional.of(new HashSet<>(this.splitString(this.conf.getProperty(COLUMN_LIST).get().toLowerCase()))) : Optional.absent(); this.clusteringKeys = this.conf.getProperty(CLUSTERING_KEYS).isPresent() ? initClusterKeys(this.splitString(this.conf.getProperty(CLUSTERING_KEYS).get())) : Collections.EMPTY_LIST; if (this.conf.getProperty(PARTITION_TYPE).isPresent()) { this.partitionType = PartitionType.valueOf(this.conf.getProperty(PARTITION_TYPE) .get().trim().toUpperCase()); } else { this.partitionType = PartitionType.NONE; } this.writeTimestamp = this.conf.getProperty(TIMESTAMP); this.timestampIsLongType = this.conf.getBooleanProperty(TIMESTAMP_IS_LONG_TYPE, false); }
@Test public void testValidConfigurationInitialized() { final CassandraSinkConfiguration conf = new CassandraSinkConfiguration(getConfig(StringTypes.EMPTY)); Assert.assertEquals(new HashSet<>(Arrays.asList("f1", "f2", "f3", "f4", "f5")), (conf.getFilteredColumns().get())); Assert.assertEquals(Arrays.asList("f1", "f2"), conf.getPartitionKeys()); Assert.assertEquals(Arrays.asList(new ClusterKey("f3", ClusterKey.Order.ASC)), conf.getClusteringKeys()); Assert.assertEquals(Arrays.asList("localhost1", "localhost2"), conf.getInitialHosts()); Assert.assertEquals("testKeyspace", conf.getKeyspace()); Assert.assertEquals("clusterFoo", conf.getClusterName()); Assert.assertEquals("testTable", conf.getTableName()); Assert.assertEquals(PartitionType.NORMAL, conf.getPartitionType()); }
@Override public void write(@NonNull final JavaRDD<AvroPayload> data) { final Configuration hadoopConf = this.conf.getHadoopConf(); converter.setKeyspaceName(conf.getKeyspace()); converter.setTableName(conf.getTableName()); final RDDWrapper<Statement> payloadWrapper = this.converter.map(data); tags.put(TABLE_NAME_TAG, this.conf.getKeyspace() + StringTypes.UNDERSCORE + this.conf.getTableName()); this.tableMetrics.get() .createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, payloadWrapper.getCount(), tags); final String clusterName = this.conf.getClusterName(); final String keyspaceName = this.conf.getKeyspace(); if (this.conf.getNativePort().isPresent()) { builder.withPort(Integer.parseInt(this.conf.getNativePort().get())); } else { builder.withPort(Integer.parseInt(CassandraSinkConfiguration.DEFAULT_OUTPUT_NATIVE_PORT)); this.conf.getInitialHosts().forEach(builder::addContactPoint);
@Override public void write(@NonNull final JavaRDD<AvroPayload> data) { final Configuration hadoopConf = this.conf.getHadoopConf(); tags.put(TABLE_NAME_TAG, this.conf.getKeyspace() + StringTypes.UNDERSCORE + this.conf.getTableName()); this.tableMetrics.get() .createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, payloadWrapper.getCount(), tags);
@Test(expected = MissingPropertyException.class) public void testMissingDataCenter() { final Configuration rawConf = getConfig(CassandraSinkConfiguration.DATACENTER); final CassandraSinkConfiguration conf = new CassandraSinkConfiguration(rawConf); conf.getHadoopConf(); Assert.fail(); }
@Test(expected = MissingPropertyException.class) public void testMissingTableName() { final Configuration rawConf = getConfig(CassandraSinkConfiguration.TABLE_NAME); final CassandraSinkConfiguration conf = new CassandraSinkConfiguration(rawConf); Assert.fail(); }
final Cluster.Builder builder = Cluster.builder().withClusterName(this.conf.getClusterName()); if (!Strings.isNullOrEmpty(ConfigHelper.getOutputKeyspaceUserName(hadoopConf)) && !Strings.isNullOrEmpty(ConfigHelper.getOutputKeyspacePassword(hadoopConf))) { ); this.conf.getInitialHosts().stream().forEach(host -> builder.addContactPoint(host)); if (this.conf.getNativePort().isPresent()) { builder.withPort(Integer.parseInt(this.conf.getNativePort().get())); } else { builder.withPort(Integer.parseInt(CassandraSinkConfiguration.DEFAULT_OUTPUT_NATIVE_PORT)); final String keySpace = this.conf.getKeyspace(); log.info("Using keyspace {}", this.conf.getKeyspace()); log.info("Using table name: {}", this.conf.getTableName()); ConfigHelper.setOutputColumnFamily(hadoopConf, this.conf.getKeyspace(), this.conf.getTableName()); this.conf.getTableName(), schemaManager.generateCreateTableStmt()); this.conf.getTableName(), schemaManager.generateInsertStmt());
@Test public void testSetHadoopSparkProperties() { final String EXECUTOR_MEM_PROP = "spark.executor.memory"; final String DRIVER_MEM_PROP = "spark.driver.memory"; final String NETWORK_TIMEOUT = "spark.network.timeout"; final String FILE_FETCH_TIMEOUT = "spark.files.fetchTimeout"; final String NUM_EXECUTORS_PROP = "spark.executor.instances"; final Configuration conf = getConfig(StringTypes.EMPTY); conf.setProperty(CassandraSinkConfiguration.HADOOP_COMMON_PREFIX + EXECUTOR_MEM_PROP, "6g"); conf.setProperty(CassandraSinkConfiguration.HADOOP_COMMON_PREFIX + DRIVER_MEM_PROP, "7g"); conf.setProperty(CassandraSinkConfiguration.HADOOP_COMMON_PREFIX + NETWORK_TIMEOUT, "1234s"); conf.setProperty(CassandraSinkConfiguration.HADOOP_COMMON_PREFIX + FILE_FETCH_TIMEOUT, "1111s"); conf.setProperty(CassandraSinkConfiguration.HADOOP_COMMON_PREFIX + NUM_EXECUTORS_PROP, "32"); final CassandraSinkConfiguration cassConf = new CassandraSinkConfiguration(conf); org.apache.hadoop.conf.Configuration hadoopConf = cassConf.getHadoopConf(); Assert.assertEquals("6g", hadoopConf.get(EXECUTOR_MEM_PROP)); Assert.assertEquals("7g", hadoopConf.get(DRIVER_MEM_PROP)); Assert.assertEquals("1234s", hadoopConf.get(NETWORK_TIMEOUT)); Assert.assertEquals("1111s", hadoopConf.get(FILE_FETCH_TIMEOUT)); Assert.assertEquals("32", hadoopConf.get(NUM_EXECUTORS_PROP)); }
@Test(expected = MissingPropertyException.class) public void testMissingClusterName() { final Configuration rawConf = getConfig(CassandraSinkConfiguration.CLUSTER_NAME); final CassandraSinkConfiguration conf = new CassandraSinkConfiguration(rawConf); Assert.fail(); }
public String getTableName() { return this.getConf().getProperty(TABLE_NAME).get().trim(); }
@Test(expected = MissingPropertyException.class) public void testMissingKeySpace() { final Configuration rawConf = getConfig(CassandraSinkConfiguration.KEYSPACE); final CassandraSinkConfiguration conf = new CassandraSinkConfiguration(rawConf); Assert.fail(); }
public String getKeyspace() { return this.getConf().getProperty(KEYSPACE).get().trim(); }
@Test(expected = MissingPropertyException.class) public void testMissingPartitionKeys() { final Configuration rawConf = getConfig(CassandraSinkConfiguration.PARTITION_KEYS); final CassandraSinkConfiguration conf = new CassandraSinkConfiguration(rawConf); Assert.fail(); }
public String getClusterName() { return this.getConf().getProperty(CLUSTER_NAME).get().trim(); }
protected CassandraSinkConfiguration initializeConfiguration(boolean excludeField, boolean hasTimestampField) { final Configuration conf = new Configuration(); conf.setProperty(CassandraSinkConfiguration.DATACENTER, "test_dc"); conf.setProperty(CassandraSinkConfiguration.DISABLE_QUERY_UNS, "true"); conf.setProperty(CassandraSinkConfiguration.INITIAL_HOSTS, "localhost"); conf.setProperty(CassandraSinkConfiguration.TABLE_NAME, CassandraTestConstants.TABLE); conf.setProperty(CassandraSinkConfiguration.KEYSPACE, CassandraTestConstants.KEY_SPACE); conf.setProperty(CassandraSinkConfiguration.CLUSTER_NAME, "testCluster"); conf.setProperty(CassandraSinkConfiguration.PARTITION_KEYS, "int_field"); conf.setProperty(CassandraSinkConfiguration.CLUSTERING_KEYS, "string_field"); // we always exclude the boolean field for now, can modify this in future to exclude a specific field if (!excludeField) { conf.setProperty(CassandraSinkConfiguration.COLUMN_LIST, Joiner.on(",").join(AvroPayloadUtil.getSchemaFields())); } else { conf.setProperty(CassandraSinkConfiguration.COLUMN_LIST, Joiner.on(",").join(AvroPayloadUtil.getSchemaFields(CassandraTestConstants.BOOLEAN_FIELD))); } conf.setProperty(CassandraSinkConfiguration.NATIVE_TRANSPORT_PORT, "9142"); return new CassandraSinkConfiguration(conf); }
public Optional<String> getUserName() { return this.getConf().getProperty(USERNAME); }
public Optional<String> getPassword() { return this.getConf().getProperty(PASSWORD); }