@Test(expected = IllegalStateException.class) public void testPartitionKeyIsNotInFieldNames() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.singletonList("non_field_name"), Collections.EMPTY_LIST); Assert.fail(); }
@Test(expected = IllegalStateException.class) public void testEmptyTableName() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, StringTypes.EMPTY, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Collections.EMPTY_LIST); Assert.fail(); }
@Test(expected = IllegalStateException.class) public void testFieldsIsNull() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, null); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Collections.EMPTY_LIST); Assert.fail(); }
@Test(expected = IllegalStateException.class) public void testFieldsIsEmpty() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, Collections.EMPTY_LIST); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Collections.EMPTY_LIST); Assert.fail(); }
@Test public void testInsertStatement() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManagerNoTTL = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Collections.EMPTY_LIST, Optional.of(10000L)); final String insertStmt = schemaManagerNoTTL.generateInsertStmt(); final String expected = "INSERT INTO marmaray.crossfit_gyms ( country_code, state_province, city, " + "capacity, gym_name ) VALUES ( ?,?,?,?,? ) USING TTL 10000"; Assert.assertEquals(expected, insertStmt); final CassandraSinkSchemaManager schemaManagerWithTTL = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Collections.EMPTY_LIST, Optional.absent()); final String insertStmtWithTTL = schemaManagerWithTTL.generateInsertStmt(); Assert.assertEquals(expected.replace("USING TTL 10000", StringTypes.EMPTY), insertStmtWithTTL); }
@Test(expected = IllegalStateException.class) public void testPartitionKeysIsEmpty() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.EMPTY_LIST, Collections.EMPTY_LIST); Assert.fail(); }
@Test(expected = IllegalStateException.class) public void tesClusterKeyIsNotInFieldNames() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.singletonList("country_code"), Collections.singletonList(new ClusterKey("non_cluster_field_name", ClusterKey.Order.ASC))); Assert.fail(); }
@Test public void testAlterTableWithOneMultipleColumns() { createBasicTable(); final List<CassandraSchemaField> joinedList = Lists.newArrayList(Iterables.concat(fields, newFields)); final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, joinedList); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final List<String> alterTableQueries = schemaManager.generateAlterTableStmt(fields.stream().map(f -> f.getFieldName()).collect(Collectors.toList())); Assert.assertTrue(alterTableQueries.size() == 2); Assert.assertEquals("ALTER TABLE marmaray.crossfit_gyms ADD new_field1 text", alterTableQueries.get(0)); Assert.assertEquals("ALTER TABLE marmaray.crossfit_gyms ADD new_field2 text", alterTableQueries.get(1)); try (final Session session = getSession()) { alterTableQueries.stream().forEach(query -> session.execute(query)); validateAlterTable(session, newFields.stream().map(f -> f.getFieldName()).collect(Collectors.toList())); } }
@Test public void testAlterTableWithOneNewColumn() { createBasicTable(); final List<CassandraSchemaField> joinedList = Lists.newArrayList(Iterables.concat(fields, Collections.singletonList(new CassandraSchemaField("new_field1", "text")))); final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, joinedList); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final List<String> alterTableQueries = schemaManager.generateAlterTableStmt(fields.stream().map(f -> f.getFieldName()).collect(Collectors.toList())); Assert.assertTrue(alterTableQueries.size() == 1); Assert.assertEquals("ALTER TABLE marmaray.crossfit_gyms ADD new_field1 text", alterTableQueries.get(0)); try (final Session session = getSession()) { alterTableQueries.stream().forEach(query -> session.execute(query)); validateAlterTable(session, Collections.singletonList("new_field1")); } }
/** * Creates a basic table in Cassandra that can be used for more testing (i.e altering) */ private void createBasicTable() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final String createTableStmt = schemaManager.generateCreateTableStmt(); try (final Session session = getSession()) { session.execute(createTableStmt); } }
@Test public void testCreateTableWithMultiplePrimaryKeys() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Collections.EMPTY_LIST); final String createTableStmt = schemaManager.generateCreateTableStmt(); final String expected = "CREATE TABLE IF NOT EXISTS marmaray.crossfit_gyms (country_code text," + "state_province text,city text,capacity int,gym_name text, PRIMARY KEY ((country_code,state_province))) "; Assert.assertEquals(expected, createTableStmt); try (final Session session = getSession()) { session.execute(createTableStmt); validateCreateTable(session); } }
@Test(expected = IllegalStateException.class) public void testPartitionAndClusterKeyHaveSameName() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("country_code", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); Assert.fail(); }
@Test(expected = IllegalStateException.class) public void testEmptyKeySpace() { final CassandraSchema schema = new CassandraSchema(StringTypes.EMPTY, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("country_code", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); Assert.fail(); }
@Test public void testCreateTableWithSinglePrimaryKeyAndNoClusteringKey() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.singletonList("country_code"), Collections.EMPTY_LIST); final String createTableStmt = schemaManager.generateCreateTableStmt(); final String expected = "CREATE TABLE IF NOT EXISTS marmaray.crossfit_gyms (country_code text," + "state_province text,city text,capacity int,gym_name text, PRIMARY KEY ((country_code))) "; Assert.assertEquals(expected, createTableStmt); try (final Session session = getSession()) { session.execute(createTableStmt); validateCreateTable(session); } }
new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys);
new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys); final CassandraSinkConfiguration conf = initializeConfiguration(true, addStringTimestamp);
private void testWriteAllFieldsMockDataToCassandra(boolean addLongTimestamp) { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestData(this.jsc.get(), 100, StringTypes.EMPTY); final List<String> schemaFields = AvroPayloadUtil.getSchemaFields(); final List<String> partitionKeys = Collections.singletonList(schemaFields.get(0)); final List<ClusterKey> clusteringKeys = Collections.singletonList( new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC)); final List<String> requiredFields = Arrays.asList(schemaFields.get(0), schemaFields.get(1)); final Optional<String> timestamp = addLongTimestamp ? Optional.of(TEST_TIMESTAMP) : Optional.absent(); final TimestampInfo tsInfo = new TimestampInfo(timestamp, true); final CassandraSinkCQLDataConverter converter = new CassandraSinkCQLDataConverter(AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY), new Configuration(), Optional.of(new HashSet<>(schemaFields)), requiredFields, tsInfo, new ErrorExtractor()); final CassandraSchemaConverter schemaConverter = new CassandraSchemaConverter(KEY_SPACE, TABLE, tsInfo, Optional.absent()); final CassandraSchema schema = schemaConverter.convertToExternalSchema( AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY)); final Optional<Long> ttl = Optional.of(10000L); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys, ttl); final CassandraSinkConfiguration conf = initializeConfiguration(false, addLongTimestamp); final CassandraClientSink sink = new CassandraClientSink(converter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, false, addLongTimestamp); }
@Test public void testCreateTableWithSinglePrimaryKeyAndOneClusteringKeyDesc() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Collections.singletonList("country_code"), Collections.singletonList(new ClusterKey("state_province", ClusterKey.Order.DESC))); final String createTableStmt = schemaManager.generateCreateTableStmt(); final String expected = "CREATE TABLE IF NOT EXISTS marmaray.crossfit_gyms (country_code text," + "state_province text,city text,capacity int,gym_name text, PRIMARY KEY ((country_code),state_province))" + " WITH CLUSTERING ORDER BY (state_province DESC)"; Assert.assertEquals(expected, createTableStmt); try (final Session session = getSession()) { session.execute(createTableStmt); validateCreateTable(session); } }
@Test public void testCreateTableWithMultiplePrimaryKeysAndClusteringKeys() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final String createTableStmt = schemaManager.generateCreateTableStmt(); final String expected = "CREATE TABLE IF NOT EXISTS marmaray.crossfit_gyms (country_code text," + "state_province text,city text,capacity int,gym_name text, PRIMARY KEY ((country_code,state_province)" + ",city,gym_name)) WITH CLUSTERING ORDER BY (city DESC,gym_name ASC)"; Assert.assertEquals(expected, createTableStmt); try (final Session session = getSession()) { session.execute(createTableStmt); validateCreateTable(session); } }
@Test public void testGenerateColumnNameQueries() { final CassandraSchema schema = new CassandraSchema(KEY_SPACE, TABLE, fields); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager( schema, Arrays.asList("country_code", "state_province"), Arrays.asList(new ClusterKey("city", ClusterKey.Order.DESC), new ClusterKey("gym_name", ClusterKey.Order.ASC))); final String cfQuery = schemaManager.getColumnNamesFromColumnFamilyQuery(); final String tableQuery = schemaManager.getColumnNamesFromTableQuery(); Assert.assertEquals("SELECT column_name FROM system_schema.columns WHERE " + "keyspace_name = 'marmaray' AND columnfamily_name = 'crossfit_gyms'", cfQuery); Assert.assertEquals("SELECT column_name FROM system_schema.columns WHERE " + "keyspace_name = 'marmaray' AND table_name = 'crossfit_gyms'", tableQuery); }