public static ClusterKey parse(@NotEmpty final String value) { final List<String> tokens = splitter.splitToList(value); Preconditions.checkState(tokens.size() <= 2); if (tokens.size() == 1) { return new ClusterKey(tokens.get(0), Order.ASC); } else { return new ClusterKey(tokens.get(0), Order.valueOf(tokens.get(1))); } } }
private List<ClusterKey> initClusterKeys(final List<String> entries) { return entries.stream().map(entry -> ClusterKey.parse(entry)).collect(Collectors.toList()); } }
/** * @return * Returns the optional clustering order syntax if it needs to be created. The clustering key should * also define if it is in ascending (default) or descending order */ private String generateClusteringOrderSyntax() { if (this.clusteringKeys.isEmpty()) { return StringTypes.EMPTY; } final List<String> clusterOrder = this.clusteringKeys.stream().map(key -> key.toString()).collect(Collectors.toList()); return String.format("WITH CLUSTERING ORDER BY (%s)", joiner.join(clusterOrder)); }
@Test(expected = IllegalStateException.class) public void testPartitionAndClusterKeyHaveSameName() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("country_code", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); Assert.fail(); }
@Test(expected = IllegalStateException.class) public void testEmptyKeySpace() { final CassandraSchema schema = new CassandraSchema(StringTypes.EMPTY, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("country_code", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); Assert.fail(); }
/** * Creates a basic table in Cassandra that can be used for more testing (i.e altering) */ private void createBasicTable() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final String createTableStmt = schemaManager.generateCreateTableStmt(); try (final Session session = getSession()) { session.execute(createTableStmt); } }
@Test(expected = IllegalStateException.class) public void tesClusterKeyIsNotInFieldNames() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.singletonList("country_code"), Collections.singletonList(new ClusterKey("non_cluster_field_name", ClusterKey.Order.ASC))); Assert.fail(); }
@Test public void testGenerateColumnNameQueries() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final String cfQuery = schemaManager.getColumnNamesFromColumnFamilyQuery(); final String tableQuery = schemaManager.getColumnNamesFromTableQuery(); Assert.assertEquals("SELECT column_name FROM system_schema.columns WHERE " + "keyspace_name = 'marmaray' AND columnfamily_name = 'crossfit_gyms'", cfQuery); Assert.assertEquals("SELECT column_name FROM system_schema.columns WHERE " + "keyspace_name = 'marmaray' AND table_name = 'crossfit_gyms'", tableQuery); }
@Test public void testAlterTableWithOneNewColumn() { createBasicTable(); final List<CassandraSchemaField> joinedList = Lists.newArrayList(Iterables.concat(fields, Collections.singletonList(new CassandraSchemaField("new_field1", "text")))); final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, joinedList); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final List<String> alterTableQueries = schemaManager.generateAlterTableStmt(fields.stream().map(f -> f.getFieldName()).collect(Collectors.toList())); Assert.assertTrue(alterTableQueries.size() == 1); Assert.assertEquals("ALTER TABLE marmaray.crossfit_gyms ADD new_field1 text", alterTableQueries.get(0)); try (final Session session = getSession()) { alterTableQueries.stream().forEach(query -> session.execute(query)); validateAlterTable(session, Collections.singletonList("new_field1")); } }
@Test public void testCreateTableWithMultiplePrimaryKeysAndClusteringKeys() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final String createTableStmt = schemaManager.generateCreateTableStmt(); final String expected = "CREATE TABLE IF NOT EXISTS marmaray.crossfit_gyms (country_code text," + "state_province text,city text,capacity int,gym_name text, PRIMARY KEY ((country_code,state_province)" + ",city,gym_name)) WITH CLUSTERING ORDER BY (city DESC,gym_name ASC)"; Assert.assertEquals(expected, createTableStmt); try (final Session session = getSession()) { session.execute(createTableStmt); validateCreateTable(session); } }
@Test public void testAlterTableWithOneMultipleColumns() { createBasicTable(); final List<CassandraSchemaField> joinedList = Lists.newArrayList(Iterables.concat(fields, newFields)); final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, joinedList); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final List<String> alterTableQueries = schemaManager.generateAlterTableStmt(fields.stream().map(f -> f.getFieldName()).collect(Collectors.toList())); Assert.assertTrue(alterTableQueries.size() == 2); Assert.assertEquals("ALTER TABLE marmaray.crossfit_gyms ADD new_field1 text", alterTableQueries.get(0)); Assert.assertEquals("ALTER TABLE marmaray.crossfit_gyms ADD new_field2 text", alterTableQueries.get(1)); try (final Session session = getSession()) { alterTableQueries.stream().forEach(query -> session.execute(query)); validateAlterTable(session, newFields.stream().map(f -> f.getFieldName()).collect(Collectors.toList())); } }
@Test public void testCreateTableWithSinglePrimaryKeyAndOneClusteringKeyDesc() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.singletonList("country_code"), Collections.singletonList(new ClusterKey("state_province", ClusterKey.Order.DESC))); final String createTableStmt = schemaManager.generateCreateTableStmt(); final String expected = "CREATE TABLE IF NOT EXISTS marmaray.crossfit_gyms (country_code text," + "state_province text,city text,capacity int,gym_name text, PRIMARY KEY ((country_code),state_province))" + " WITH CLUSTERING ORDER BY (state_province DESC)"; Assert.assertEquals(expected, createTableStmt); try (final Session session = getSession()) { session.execute(createTableStmt); validateCreateTable(session); } }
new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC));
private void testWriteAllFieldsMockDataToCassandra(boolean addLongTimestamp) { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestData(this.jsc.get(), 100, StringTypes.EMPTY); final List<String> schemaFields = AvroPayloadUtil.getSchemaFields(); final List<String> partitionKeys = Collections.singletonList(schemaFields.get(0)); final List<ClusterKey> clusteringKeys = Collections.singletonList( new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC)); final List<String> requiredFields = Arrays.asList(schemaFields.get(0), schemaFields.get(1)); final Optional<String> timestamp = addLongTimestamp ? Optional.of(TEST_TIMESTAMP) : Optional.absent(); final TimestampInfo tsInfo = new TimestampInfo(timestamp, true); final CassandraSinkDataConverter dataconverter = new CassandraSinkDataConverter(AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY), new Configuration(), Optional.of(new HashSet<>(schemaFields)), requiredFields, tsInfo, new ErrorExtractor()); final CassandraSchemaConverter schemaConverter = new CassandraSchemaConverter(KEY_SPACE, TABLE, tsInfo, Optional.absent()); final CassandraSchema schema = schemaConverter.convertToExternalSchema( AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY)); final Optional<Long> ttl = Optional.of(10000L); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys, ttl); final CassandraSinkConfiguration conf = initializeConfiguration(false, addLongTimestamp); final CassandraSSTableSink sink = new CassandraSSTableSink(dataconverter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, false, addLongTimestamp); }
private void testWriteAllFieldsMockDataToCassandra(boolean addLongTimestamp) { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestData(this.jsc.get(), 100, StringTypes.EMPTY); final List<String> schemaFields = AvroPayloadUtil.getSchemaFields(); final List<String> partitionKeys = Collections.singletonList(schemaFields.get(0)); final List<ClusterKey> clusteringKeys = Collections.singletonList( new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC)); final List<String> requiredFields = Arrays.asList(schemaFields.get(0), schemaFields.get(1)); final Optional<String> timestamp = addLongTimestamp ? Optional.of(TEST_TIMESTAMP) : Optional.absent(); final TimestampInfo tsInfo = new TimestampInfo(timestamp, true); final CassandraSinkCQLDataConverter converter = new CassandraSinkCQLDataConverter(AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY), new Configuration(), Optional.of(new HashSet<>(schemaFields)), requiredFields, tsInfo, new ErrorExtractor()); final CassandraSchemaConverter schemaConverter = new CassandraSchemaConverter(KEY_SPACE, TABLE, tsInfo, Optional.absent()); final CassandraSchema schema = schemaConverter.convertToExternalSchema( AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY)); final Optional<Long> ttl = Optional.of(10000L); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys, ttl); final CassandraSinkConfiguration conf = initializeConfiguration(false, addLongTimestamp); final CassandraClientSink sink = new CassandraClientSink(converter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, false, addLongTimestamp); }
new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC));
new ClusterKey(CassandraTestConstants.STRING_FIELD, ClusterKey.Order.DESC));