/** * We exclude the boolean field from the input schema and don't write it to Cassandra but do add a timestamp */ @Test public void testWriteOnlySpecifiedFieldsMockDataToCassandraWithTimestamp() { testWriteOnlySpecifiedFieldsMockDataToCassandra(true); }
/** * We exclude the boolean field from the input schema and don't write it to Cassandra and do NOT add a timestamp */ @Test public void testWriteOnlySpecifiedFieldsMockDataToCassandraWithoutTimestamp() { testWriteOnlySpecifiedFieldsMockDataToCassandra(false); }
@Test public void testWriteAllFieldsMockDataToCassandraWithTimestamp() { testWriteAllFieldsMockDataToCassandra(true); }
final CassandraSinkConfiguration conf = initializeConfiguration(true, addStringTimestamp); final CassandraSSTableSink sink = new CassandraSSTableSink(converter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, true, addStringTimestamp);
new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys); final CassandraSinkConfiguration conf = initializeConfiguration(true, addStringTimestamp); final CassandraClientSink sink = new CassandraClientSink(converter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, true, addStringTimestamp);
this.setupCassandraTable(hadoopConf); this.initSystemProperties(cassandraRecords); cassandraRecords.mapToPair(f -> new Tuple2<>(generateDummyValue(), f.convertData()));
new CassandraSinkSchemaManager(cassandraSchema, partitionKeys, clusteringKeys); final CassandraSinkConfiguration conf = initializeConfiguration(false, includeTimestamp); final CassandraClientSink sink = new CassandraClientSink(converter, schemaManager, conf); try { sink.write(testData); } catch (Exception e) { Assert.assertEquals(SparkException.class, e.getClass());
final CassandraSinkConfiguration conf = initializeConfiguration(false, includeTimestamp); final CassandraSSTableSink sink = new CassandraSSTableSink(converter, schemaManager, conf); try { sink.write(testData); } catch (Exception e) { Assert.assertEquals(SparkException.class, e.getClass());
@Test public void testWriteAllFieldsMockDataToCassandraWithTimestamp() { testWriteAllFieldsMockDataToCassandra(true); }
@After public void teardownTest() { super.teardownTest(); CassandraTestUtil.teardownCluster(); }
@Before public void setupTest() { super.setupTest(); try { CassandraTestUtil.setupCluster(); } catch (TTransportException | IOException e) { throw new JobRuntimeException("Error while initializing Cassandra cluster for test: " + e.getMessage()); } }
this.setupCassandraTable(hadoopConf);
private void testWriteAllFieldsMockDataToCassandra(boolean addLongTimestamp) { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestData(this.jsc.get(), 100, StringTypes.EMPTY); final List<String> schemaFields = AvroPayloadUtil.getSchemaFields(); final List<String> partitionKeys = Collections.singletonList(schemaFields.get(0)); final List<ClusterKey> clusteringKeys = Collections.singletonList( new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC)); final List<String> requiredFields = Arrays.asList(schemaFields.get(0), schemaFields.get(1)); final Optional<String> timestamp = addLongTimestamp ? Optional.of(TEST_TIMESTAMP) : Optional.absent(); final TimestampInfo tsInfo = new TimestampInfo(timestamp, true); final CassandraSinkDataConverter dataconverter = new CassandraSinkDataConverter(AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY), new Configuration(), Optional.of(new HashSet<>(schemaFields)), requiredFields, tsInfo, new ErrorExtractor()); final CassandraSchemaConverter schemaConverter = new CassandraSchemaConverter(KEY_SPACE, TABLE, tsInfo, Optional.absent()); final CassandraSchema schema = schemaConverter.convertToExternalSchema( AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY)); final Optional<Long> ttl = Optional.of(10000L); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys, ttl); final CassandraSinkConfiguration conf = initializeConfiguration(false, addLongTimestamp); final CassandraSSTableSink sink = new CassandraSSTableSink(dataconverter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, false, addLongTimestamp); }
private void testWriteAllFieldsMockDataToCassandra(boolean addLongTimestamp) { final JavaRDD<AvroPayload> testData = AvroPayloadUtil.generateTestData(this.jsc.get(), 100, StringTypes.EMPTY); final List<String> schemaFields = AvroPayloadUtil.getSchemaFields(); final List<String> partitionKeys = Collections.singletonList(schemaFields.get(0)); final List<ClusterKey> clusteringKeys = Collections.singletonList( new ClusterKey(schemaFields.get(1), ClusterKey.Order.DESC)); final List<String> requiredFields = Arrays.asList(schemaFields.get(0), schemaFields.get(1)); final Optional<String> timestamp = addLongTimestamp ? Optional.of(TEST_TIMESTAMP) : Optional.absent(); final TimestampInfo tsInfo = new TimestampInfo(timestamp, true); final CassandraSinkCQLDataConverter converter = new CassandraSinkCQLDataConverter(AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY), new Configuration(), Optional.of(new HashSet<>(schemaFields)), requiredFields, tsInfo, new ErrorExtractor()); final CassandraSchemaConverter schemaConverter = new CassandraSchemaConverter(KEY_SPACE, TABLE, tsInfo, Optional.absent()); final CassandraSchema schema = schemaConverter.convertToExternalSchema( AvroPayloadUtil.getAvroTestDataSchema(StringTypes.EMPTY)); final Optional<Long> ttl = Optional.of(10000L); final CassandraSinkSchemaManager schemaManager = new CassandraSinkSchemaManager(schema, partitionKeys, clusteringKeys, ttl); final CassandraSinkConfiguration conf = initializeConfiguration(false, addLongTimestamp); final CassandraClientSink sink = new CassandraClientSink(converter, schemaManager, conf); sink.write(testData); validateCassandraTable(100, false, addLongTimestamp); }
/** * We exclude the boolean field from the input schema and don't write it to Cassandra and do NOT add a timestamp */ @Test public void testWriteOnlySpecifiedFieldsMockDataToCassandraWithoutTimestamp() { testWriteOnlySpecifiedFieldsMockDataToCassandra(false); }
/** * We exclude the boolean field from the input schema and don't write it to Cassandra but do add a timestamp */ @Test public void testWriteOnlySpecifiedFieldsMockDataToCassandraWithTimestamp() { testWriteOnlySpecifiedFieldsMockDataToCassandra(true); }
@Test public void testWriteAllFieldsMockDataToCassandraWithoutTimestamp() { testWriteAllFieldsMockDataToCassandra(false); }
@Test public void testWriteAllFieldsMockDataToCassandraWithoutTimestamp() { testWriteAllFieldsMockDataToCassandra(false); }
@After public void teardownTest() { super.teardownTest(); CassandraTestUtil.teardownCluster(); }
@Before public void setupTest() { super.setupTest(); try { CassandraTestUtil.setupCluster(); } catch (TTransportException | IOException e) { throw new JobRuntimeException("Error while initializing Cassandra cluster for test: " + e.getMessage()); } }